diff options
author | shmel1k <shmel1k@ydb.tech> | 2023-11-26 18:16:14 +0300 |
---|---|---|
committer | shmel1k <shmel1k@ydb.tech> | 2023-11-26 18:43:30 +0300 |
commit | b8cf9e88f4c5c64d9406af533d8948deb050d695 (patch) | |
tree | 218eb61fb3c3b96ec08b4d8cdfef383104a87d63 /contrib/python/Twisted/py2/twisted/runner | |
parent | 523f645a83a0ec97a0332dbc3863bb354c92a328 (diff) | |
download | ydb-b8cf9e88f4c5c64d9406af533d8948deb050d695.tar.gz |
add kikimr_configure
Diffstat (limited to 'contrib/python/Twisted/py2/twisted/runner')
6 files changed, 882 insertions, 0 deletions
diff --git a/contrib/python/Twisted/py2/twisted/runner/__init__.py b/contrib/python/Twisted/py2/twisted/runner/__init__.py new file mode 100644 index 0000000000..024a284959 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted Runner: Run and monitor processes. +""" diff --git a/contrib/python/Twisted/py2/twisted/runner/inetd.py b/contrib/python/Twisted/py2/twisted/runner/inetd.py new file mode 100644 index 0000000000..3402e232e4 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetd.py @@ -0,0 +1,70 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +# + +""" +Twisted inetd. + +Maintainer: Andrew Bennetts + +Future Plans: Bugfixes. Specifically for UDP and Sun-RPC, which don't work +correctly yet. +""" + +import os + +from twisted.internet import process, reactor, fdesc +from twisted.internet.protocol import Protocol, ServerFactory +from twisted.protocols import wire + +# A dict of known 'internal' services (i.e. those that don't involve spawning +# another process. +internalProtocols = { + 'echo': wire.Echo, + 'chargen': wire.Chargen, + 'discard': wire.Discard, + 'daytime': wire.Daytime, + 'time': wire.Time, +} + + +class InetdProtocol(Protocol): + """Forks a child process on connectionMade, passing the socket as fd 0.""" + def connectionMade(self): + sockFD = self.transport.fileno() + childFDs = {0: sockFD, 1: sockFD} + if self.factory.stderrFile: + childFDs[2] = self.factory.stderrFile.fileno() + + # processes run by inetd expect blocking sockets + # FIXME: maybe this should be done in process.py? are other uses of + # Process possibly affected by this? + fdesc.setBlocking(sockFD) + if 2 in childFDs: + fdesc.setBlocking(childFDs[2]) + + service = self.factory.service + uid = service.user + gid = service.group + + # don't tell Process to change our UID/GID if it's what we + # already are + if uid == os.getuid(): + uid = None + if gid == os.getgid(): + gid = None + + process.Process(None, service.program, service.programArgs, os.environ, + None, None, uid, gid, childFDs) + + reactor.removeReader(self.transport) + reactor.removeWriter(self.transport) + + +class InetdFactory(ServerFactory): + protocol = InetdProtocol + stderrFile = None + + def __init__(self, service): + self.service = service diff --git a/contrib/python/Twisted/py2/twisted/runner/inetdconf.py b/contrib/python/Twisted/py2/twisted/runner/inetdconf.py new file mode 100644 index 0000000000..391f2e0573 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetdconf.py @@ -0,0 +1,198 @@ +# -*- test-case-name: twisted.runner.test.test_inetdconf -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Parser for inetd.conf files +""" + +# Various exceptions +class InvalidConfError(Exception): + """ + Invalid configuration file + """ + + + +class InvalidInetdConfError(InvalidConfError): + """ + Invalid inetd.conf file + """ + + + +class InvalidServicesConfError(InvalidConfError): + """ + Invalid services file + """ + + + +class UnknownService(Exception): + """ + Unknown service name + """ + + + +class SimpleConfFile: + """ + Simple configuration file parser superclass. + + Filters out comments and empty lines (which includes lines that only + contain comments). + + To use this class, override parseLine or parseFields. + """ + + commentChar = '#' + defaultFilename = None + + def parseFile(self, file=None): + """ + Parse a configuration file + + If file is None and self.defaultFilename is set, it will open + defaultFilename and use it. + """ + close = False + if file is None and self.defaultFilename: + file = open(self.defaultFilename,'r') + close = True + + try: + for line in file.readlines(): + # Strip out comments + comment = line.find(self.commentChar) + if comment != -1: + line = line[:comment] + + # Strip whitespace + line = line.strip() + + # Skip empty lines (and lines which only contain comments) + if not line: + continue + + self.parseLine(line) + finally: + if close: + file.close() + + + def parseLine(self, line): + """ + Override this. + + By default, this will split the line on whitespace and call + self.parseFields (catching any errors). + """ + try: + self.parseFields(*line.split()) + except ValueError: + raise InvalidInetdConfError('Invalid line: ' + repr(line)) + + + def parseFields(self, *fields): + """ + Override this. + """ + + + +class InetdService: + """ + A simple description of an inetd service. + """ + name = None + port = None + socketType = None + protocol = None + wait = None + user = None + group = None + program = None + programArgs = None + + def __init__(self, name, port, socketType, protocol, wait, user, group, + program, programArgs): + self.name = name + self.port = port + self.socketType = socketType + self.protocol = protocol + self.wait = wait + self.user = user + self.group = group + self.program = program + self.programArgs = programArgs + + + +class InetdConf(SimpleConfFile): + """ + Configuration parser for a traditional UNIX inetd(8) + """ + + defaultFilename = '/etc/inetd.conf' + + def __init__(self, knownServices=None): + self.services = [] + + if knownServices is None: + knownServices = ServicesConf() + knownServices.parseFile() + self.knownServices = knownServices + + + def parseFields(self, serviceName, socketType, protocol, wait, user, + program, *programArgs): + """ + Parse an inetd.conf file. + + Implemented from the description in the Debian inetd.conf man page. + """ + # Extract user (and optional group) + user, group = (user.split('.') + [None])[:2] + + # Find the port for a service + port = self.knownServices.services.get((serviceName, protocol), None) + if not port and not protocol.startswith('rpc/'): + # FIXME: Should this be discarded/ignored, rather than throwing + # an exception? + try: + port = int(serviceName) + serviceName = 'unknown' + except: + raise UnknownService("Unknown service: %s (%s)" % ( + serviceName, protocol)) + + self.services.append(InetdService(serviceName, port, socketType, + protocol, wait, user, group, program, + programArgs)) + + + +class ServicesConf(SimpleConfFile): + """ + /etc/services parser + + @ivar services: dict mapping service names to (port, protocol) tuples. + """ + + defaultFilename = '/etc/services' + + def __init__(self): + self.services = {} + + + def parseFields(self, name, portAndProtocol, *aliases): + try: + port, protocol = portAndProtocol.split('/') + port = int(port) + except: + raise InvalidServicesConfError( + 'Invalid port/protocol: %s' % (repr(portAndProtocol),)) + + self.services[(name, protocol)] = port + for alias in aliases: + self.services[(alias, protocol)] = port diff --git a/contrib/python/Twisted/py2/twisted/runner/inetdtap.py b/contrib/python/Twisted/py2/twisted/runner/inetdtap.py new file mode 100644 index 0000000000..b959942c51 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/inetdtap.py @@ -0,0 +1,109 @@ +# -*- test-case-name: twisted.runner.test.test_inetdtap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Twisted inetd TAP support + +The purpose of inetdtap is to provide an inetd-like server, to allow Twisted to +invoke other programs to handle incoming sockets. +This is a useful thing as a "networking swiss army knife" tool, like netcat. +""" + +import pwd, grp, socket + +from twisted.runner import inetd, inetdconf +from twisted.python import log, usage +from twisted.internet.protocol import ServerFactory +from twisted.application import internet, service as appservice + +# Protocol map +protocolDict = {'tcp': socket.IPPROTO_TCP, 'udp': socket.IPPROTO_UDP} + + +class Options(usage.Options): + """ + To use it, create a file named `sample-inetd.conf` with: + + 8123 stream tcp wait some_user /bin/cat - + + You can then run it as in the following example and port 8123 became an + echo server. + + twistd -n inetd -f sample-inetd.conf + """ + + optParameters = [ + ['rpc', 'r', '/etc/rpc', 'DEPRECATED. RPC procedure table file'], + ['file', 'f', '/etc/inetd.conf', 'Service configuration file'] + ] + + optFlags = [['nointernal', 'i', "Don't run internal services"]] + + compData = usage.Completions( + optActions={"file": usage.CompleteFiles('*.conf')} + ) + + + +def makeService(config): + s = appservice.MultiService() + conf = inetdconf.InetdConf() + with open(config['file']) as f: + conf.parseFile(f) + + for service in conf.services: + protocol = service.protocol + + if service.protocol.startswith('rpc/'): + log.msg('Skipping rpc service due to lack of rpc support') + continue + + if (protocol, service.socketType) not in [('tcp', 'stream'), + ('udp', 'dgram')]: + log.msg('Skipping unsupported type/protocol: %s/%s' + % (service.socketType, service.protocol)) + continue + + # Convert the username into a uid (if necessary) + try: + service.user = int(service.user) + except ValueError: + try: + service.user = pwd.getpwnam(service.user)[2] + except KeyError: + log.msg('Unknown user: ' + service.user) + continue + + # Convert the group name into a gid (if necessary) + if service.group is None: + # If no group was specified, use the user's primary group + service.group = pwd.getpwuid(service.user)[3] + else: + try: + service.group = int(service.group) + except ValueError: + try: + service.group = grp.getgrnam(service.group)[2] + except KeyError: + log.msg('Unknown group: ' + service.group) + continue + + if service.program == 'internal': + if config['nointernal']: + continue + + # Internal services can use a standard ServerFactory + if service.name not in inetd.internalProtocols: + log.msg('Unknown internal service: ' + service.name) + continue + factory = ServerFactory() + factory.protocol = inetd.internalProtocols[service.name] + else: + factory = inetd.InetdFactory(service) + + if protocol == 'tcp': + internet.TCPServer(service.port, factory).setServiceParent(s) + elif protocol == 'udp': + raise RuntimeError("not supporting UDP") + return s diff --git a/contrib/python/Twisted/py2/twisted/runner/procmon.py b/contrib/python/Twisted/py2/twisted/runner/procmon.py new file mode 100644 index 0000000000..05909570b1 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/procmon.py @@ -0,0 +1,426 @@ +# -*- test-case-name: twisted.runner.test.test_procmon -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for starting, monitoring, and restarting child process. +""" +import attr +import incremental + +from twisted.python import deprecate +from twisted.internet import error, protocol, reactor as _reactor +from twisted.application import service +from twisted.protocols import basic +from twisted.logger import Logger + + + +@attr.s(frozen=True) +class _Process(object): + """ + The parameters of a process to be restarted. + + @ivar args: command-line arguments (including name of command as first one) + @type args: C{list} + + @ivar uid: user-id to run process as, or None (which means inherit uid) + @type uid: C{int} + + @ivar gid: group-id to run process as, or None (which means inherit gid) + @type gid: C{int} + + @ivar env: environment for process + @type env: C{dict} + + @ivar cwd: initial working directory for process or None + (which means inherit cwd) + @type cwd: C{str} + """ + + args = attr.ib() + uid = attr.ib(default=None) + gid = attr.ib(default=None) + env = attr.ib(default=attr.Factory(dict)) + cwd = attr.ib(default=None) + + @deprecate.deprecated(incremental.Version('Twisted', 18, 7, 0)) + def toTuple(self): + """ + Convert process to tuple. + + Convert process to tuple that looks like the legacy structure + of processes, for potential users who inspected processes + directly. + + This was only an accidental feature, and will be removed. If + you need to remember what processes were added to a process monitor, + keep track of that when they are added. The process list + inside the process monitor is no longer a public API. + + This allows changing the internal structure of the process list, + when warranted by bug fixes or additional features. + + @return: tuple representation of process + """ + return (self.args, self.uid, self.gid, self.env) + + + +class DummyTransport: + + disconnecting = 0 + + + +transport = DummyTransport() + + + +class LineLogger(basic.LineReceiver): + + tag = None + stream = None + delimiter = b'\n' + service = None + + def lineReceived(self, line): + try: + line = line.decode('utf-8') + except UnicodeDecodeError: + line = repr(line) + + self.service.log.info(u'[{tag}] {line}', + tag=self.tag, + line=line, + stream=self.stream) + + + +class LoggingProtocol(protocol.ProcessProtocol): + + service = None + name = None + + def connectionMade(self): + self._output = LineLogger() + self._output.tag = self.name + self._output.stream = 'stdout' + self._output.service = self.service + self._outputEmpty = True + + self._error = LineLogger() + self._error.tag = self.name + self._error.stream = 'stderr' + self._error.service = self.service + self._errorEmpty = True + + self._output.makeConnection(transport) + self._error.makeConnection(transport) + + + def outReceived(self, data): + self._output.dataReceived(data) + self._outputEmpty = data[-1] == b'\n' + + def errReceived(self, data): + self._error.dataReceived(data) + self._errorEmpty = data[-1] == b'\n' + + def processEnded(self, reason): + if not self._outputEmpty: + self._output.dataReceived(b'\n') + if not self._errorEmpty: + self._error.dataReceived(b'\n') + self.service.connectionLost(self.name) + + @property + def output(self): + return self._output + + @property + def empty(self): + return self._outputEmpty + + + +class ProcessMonitor(service.Service): + """ + ProcessMonitor runs processes, monitors their progress, and restarts + them when they die. + + The ProcessMonitor will not attempt to restart a process that appears to + die instantly -- with each "instant" death (less than 1 second, by + default), it will delay approximately twice as long before restarting + it. A successful run will reset the counter. + + The primary interface is L{addProcess} and L{removeProcess}. When the + service is running (that is, when the application it is attached to is + running), adding a process automatically starts it. + + Each process has a name. This name string must uniquely identify the + process. In particular, attempting to add two processes with the same + name will result in a C{KeyError}. + + @type threshold: C{float} + @ivar threshold: How long a process has to live before the death is + considered instant, in seconds. The default value is 1 second. + + @type killTime: C{float} + @ivar killTime: How long a process being killed has to get its affairs + in order before it gets killed with an unmaskable signal. The + default value is 5 seconds. + + @type minRestartDelay: C{float} + @ivar minRestartDelay: The minimum time (in seconds) to wait before + attempting to restart a process. Default 1s. + + @type maxRestartDelay: C{float} + @ivar maxRestartDelay: The maximum time (in seconds) to wait before + attempting to restart a process. Default 3600s (1h). + + @type _reactor: L{IReactorProcess} provider + @ivar _reactor: A provider of L{IReactorProcess} and L{IReactorTime} + which will be used to spawn processes and register delayed calls. + + @type log: L{Logger} + @ivar log: The logger used to propagate log messages from spawned + processes. + + """ + threshold = 1 + killTime = 5 + minRestartDelay = 1 + maxRestartDelay = 3600 + log = Logger() + + + def __init__(self, reactor=_reactor): + self._reactor = reactor + + self._processes = {} + self.protocols = {} + self.delay = {} + self.timeStarted = {} + self.murder = {} + self.restart = {} + + + @deprecate.deprecatedProperty(incremental.Version('Twisted', 18, 7, 0)) + def processes(self): + """ + Processes as dict of tuples + + @return: Dict of process name to monitored processes as tuples + """ + return {name: process.toTuple() + for name, process in self._processes.items()} + + + @deprecate.deprecated(incremental.Version('Twisted', 18, 7, 0)) + def __getstate__(self): + dct = service.Service.__getstate__(self) + del dct['_reactor'] + dct['protocols'] = {} + dct['delay'] = {} + dct['timeStarted'] = {} + dct['murder'] = {} + dct['restart'] = {} + del dct['_processes'] + dct['processes'] = self.processes + return dct + + + def addProcess(self, name, args, uid=None, gid=None, env={}, cwd=None): + """ + Add a new monitored process and start it immediately if the + L{ProcessMonitor} service is running. + + Note that args are passed to the system call, not to the shell. If + running the shell is desired, the common idiom is to use + C{ProcessMonitor.addProcess("name", ['/bin/sh', '-c', shell_script])} + + @param name: A name for this process. This value must be + unique across all processes added to this monitor. + @type name: C{str} + @param args: The argv sequence for the process to launch. + @param uid: The user ID to use to run the process. If L{None}, + the current UID is used. + @type uid: C{int} + @param gid: The group ID to use to run the process. If L{None}, + the current GID is used. + @type uid: C{int} + @param env: The environment to give to the launched process. See + L{IReactorProcess.spawnProcess}'s C{env} parameter. + @type env: C{dict} + @param cwd: The initial working directory of the launched process. + The default of C{None} means inheriting the laucnhing process's + working directory. + @type env: C{dict} + @raises: C{KeyError} if a process with the given name already + exists + """ + if name in self._processes: + raise KeyError("remove %s first" % (name,)) + self._processes[name] = _Process(args, uid, gid, env, cwd) + self.delay[name] = self.minRestartDelay + if self.running: + self.startProcess(name) + + + def removeProcess(self, name): + """ + Stop the named process and remove it from the list of monitored + processes. + + @type name: C{str} + @param name: A string that uniquely identifies the process. + """ + self.stopProcess(name) + del self._processes[name] + + + def startService(self): + """ + Start all monitored processes. + """ + service.Service.startService(self) + for name in list(self._processes): + self.startProcess(name) + + + def stopService(self): + """ + Stop all monitored processes and cancel all scheduled process restarts. + """ + service.Service.stopService(self) + + # Cancel any outstanding restarts + for name, delayedCall in list(self.restart.items()): + if delayedCall.active(): + delayedCall.cancel() + + for name in list(self._processes): + self.stopProcess(name) + + + def connectionLost(self, name): + """ + Called when a monitored processes exits. If + L{service.IService.running} is L{True} (ie the service is started), the + process will be restarted. + If the process had been running for more than + L{ProcessMonitor.threshold} seconds it will be restarted immediately. + If the process had been running for less than + L{ProcessMonitor.threshold} seconds, the restart will be delayed and + each time the process dies before the configured threshold, the restart + delay will be doubled - up to a maximum delay of maxRestartDelay sec. + + @type name: C{str} + @param name: A string that uniquely identifies the process + which exited. + """ + # Cancel the scheduled _forceStopProcess function if the process + # dies naturally + if name in self.murder: + if self.murder[name].active(): + self.murder[name].cancel() + del self.murder[name] + + del self.protocols[name] + + if self._reactor.seconds() - self.timeStarted[name] < self.threshold: + # The process died too fast - backoff + nextDelay = self.delay[name] + self.delay[name] = min(self.delay[name] * 2, self.maxRestartDelay) + + else: + # Process had been running for a significant amount of time + # restart immediately + nextDelay = 0 + self.delay[name] = self.minRestartDelay + + # Schedule a process restart if the service is running + if self.running and name in self._processes: + self.restart[name] = self._reactor.callLater(nextDelay, + self.startProcess, + name) + + + def startProcess(self, name): + """ + @param name: The name of the process to be started + """ + # If a protocol instance already exists, it means the process is + # already running + if name in self.protocols: + return + + process = self._processes[name] + + proto = LoggingProtocol() + proto.service = self + proto.name = name + self.protocols[name] = proto + self.timeStarted[name] = self._reactor.seconds() + self._reactor.spawnProcess(proto, process.args[0], process.args, + uid=process.uid, gid=process.gid, + env=process.env, path=process.cwd) + + + def _forceStopProcess(self, proc): + """ + @param proc: An L{IProcessTransport} provider + """ + try: + proc.signalProcess('KILL') + except error.ProcessExitedAlready: + pass + + + def stopProcess(self, name): + """ + @param name: The name of the process to be stopped + """ + if name not in self._processes: + raise KeyError('Unrecognized process name: %s' % (name,)) + + proto = self.protocols.get(name, None) + if proto is not None: + proc = proto.transport + try: + proc.signalProcess('TERM') + except error.ProcessExitedAlready: + pass + else: + self.murder[name] = self._reactor.callLater( + self.killTime, + self._forceStopProcess, proc) + + + def restartAll(self): + """ + Restart all processes. This is useful for third party management + services to allow a user to restart servers because of an outside change + in circumstances -- for example, a new version of a library is + installed. + """ + for name in self._processes: + self.stopProcess(name) + + + def __repr__(self): + l = [] + for name, proc in self._processes.items(): + uidgid = '' + if proc.uid is not None: + uidgid = str(proc.uid) + if proc.gid is not None: + uidgid += ':'+str(proc.gid) + + if uidgid: + uidgid = '(' + uidgid + ')' + l.append('%r%s: %r' % (name, uidgid, proc.args)) + return ('<' + self.__class__.__name__ + ' ' + + ' '.join(l) + + '>') diff --git a/contrib/python/Twisted/py2/twisted/runner/procmontap.py b/contrib/python/Twisted/py2/twisted/runner/procmontap.py new file mode 100644 index 0000000000..c0e72a45e8 --- /dev/null +++ b/contrib/python/Twisted/py2/twisted/runner/procmontap.py @@ -0,0 +1,73 @@ +# -*- test-case-name: twisted.runner.test.test_procmontap -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Support for creating a service which runs a process monitor. +""" + +from twisted.python import usage +from twisted.runner.procmon import ProcessMonitor + + +class Options(usage.Options): + """ + Define the options accepted by the I{twistd procmon} plugin. + """ + + synopsis = "[procmon options] commandline" + + optParameters = [["threshold", "t", 1, "How long a process has to live " + "before the death is considered instant, in seconds.", + float], + ["killtime", "k", 5, "How long a process being killed " + "has to get its affairs in order before it gets killed " + "with an unmaskable signal.", + float], + ["minrestartdelay", "m", 1, "The minimum time (in " + "seconds) to wait before attempting to restart a " + "process", float], + ["maxrestartdelay", "M", 3600, "The maximum time (in " + "seconds) to wait before attempting to restart a " + "process", float]] + + optFlags = [] + + + longdesc = """\ +procmon runs processes, monitors their progress, and restarts them when they +die. + +procmon will not attempt to restart a process that appears to die instantly; +with each "instant" death (less than 1 second, by default), it will delay +approximately twice as long before restarting it. A successful run will reset +the counter. + +Eg twistd procmon sleep 10""" + + def parseArgs(self, *args): + """ + Grab the command line that is going to be started and monitored + """ + self['args'] = args + + + def postOptions(self): + """ + Check for dependencies. + """ + if len(self["args"]) < 1: + raise usage.UsageError("Please specify a process commandline") + + + +def makeService(config): + s = ProcessMonitor() + + s.threshold = config["threshold"] + s.killTime = config["killtime"] + s.minRestartDelay = config["minrestartdelay"] + s.maxRestartDelay = config["maxrestartdelay"] + + s.addProcess(" ".join(config["args"]), config["args"]) + return s |