diff options
| author | iddqd <[email protected]> | 2024-06-17 19:25:38 +0300 |
|---|---|---|
| committer | iddqd <[email protected]> | 2024-06-17 19:35:02 +0300 |
| commit | e6ca8ab46f09c2860e3e1915ccadc1a02c6b7a6a (patch) | |
| tree | 0751dd161bdaaca771ed7d44e2827cf844c0fa0c /contrib/python | |
| parent | dd1584c30516e4bb676349a9c7e73a226349b0d1 (diff) | |
Revert commit rXXXXXX, add contrib/python/yandexcloud to import
41a6b51d2f72f0308cd2979b07b9262a6d3ef8f5
Diffstat (limited to 'contrib/python')
37 files changed, 0 insertions, 3923 deletions
diff --git a/contrib/python/PyJWT/py2/.dist-info/METADATA b/contrib/python/PyJWT/py2/.dist-info/METADATA deleted file mode 100644 index 47ee5589077..00000000000 --- a/contrib/python/PyJWT/py2/.dist-info/METADATA +++ /dev/null @@ -1,115 +0,0 @@ -Metadata-Version: 2.1 -Name: PyJWT -Version: 1.7.1 -Summary: JSON Web Token implementation in Python -Home-page: http://github.com/jpadilla/pyjwt -Author: Jose Padilla -Author-email: [email protected] -License: MIT -Keywords: jwt json web token security signing -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Natural Language :: English -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Topic :: Utilities -Provides-Extra: crypto -Requires-Dist: cryptography (>=1.4) ; extra == 'crypto' -Provides-Extra: flake8 -Requires-Dist: flake8 ; extra == 'flake8' -Requires-Dist: flake8-import-order ; extra == 'flake8' -Requires-Dist: pep8-naming ; extra == 'flake8' -Provides-Extra: test -Requires-Dist: pytest (<5.0.0,>=4.0.1) ; extra == 'test' -Requires-Dist: pytest-cov (<3.0.0,>=2.6.0) ; extra == 'test' -Requires-Dist: pytest-runner (<5.0.0,>=4.2) ; extra == 'test' - -PyJWT -===== - -.. image:: https://travis-ci.com/jpadilla/pyjwt.svg?branch=master - :target: http://travis-ci.com/jpadilla/pyjwt?branch=master - -.. image:: https://ci.appveyor.com/api/projects/status/h8nt70aqtwhht39t?svg=true - :target: https://ci.appveyor.com/project/jpadilla/pyjwt - -.. image:: https://img.shields.io/pypi/v/pyjwt.svg - :target: https://pypi.python.org/pypi/pyjwt - -.. image:: https://coveralls.io/repos/jpadilla/pyjwt/badge.svg?branch=master - :target: https://coveralls.io/r/jpadilla/pyjwt?branch=master - -.. image:: https://readthedocs.org/projects/pyjwt/badge/?version=latest - :target: https://pyjwt.readthedocs.io - -A Python implementation of `RFC 7519 <https://tools.ietf.org/html/rfc7519>`_. Original implementation was written by `@progrium <https://github.com/progrium>`_. - -Sponsor -------- - -+--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| |auth0-logo| | If you want to quickly add secure token-based authentication to Python projects, feel free to check Auth0's Python SDK and free plan at `auth0.com/overview <https://auth0.com/overview?utm_source=GHsponsor&utm_medium=GHsponsor&utm_campaign=pyjwt&utm_content=auth>`_. | -+--------------+-----------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -.. |auth0-logo| image:: https://user-images.githubusercontent.com/83319/31722733-de95bbde-b3ea-11e7-96bf-4f4e8f915588.png - -Installing ----------- - -Install with **pip**: - -.. code-block:: sh - - $ pip install PyJWT - - -Usage ------ - -.. code:: python - - >>> import jwt - >>> encoded = jwt.encode({'some': 'payload'}, 'secret', algorithm='HS256') - 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzb21lIjoicGF5bG9hZCJ9.4twFt5NiznN84AWoo1d7KO1T_yoc0Z6XOpOVswacPZg' - - >>> jwt.decode(encoded, 'secret', algorithms=['HS256']) - {'some': 'payload'} - - -Command line ------------- - -Usage:: - - pyjwt [options] INPUT - -Decoding examples:: - - pyjwt --key=secret decode TOKEN - pyjwt decode --no-verify TOKEN - -See more options executing ``pyjwt --help``. - - -Documentation -------------- - -View the full docs online at https://pyjwt.readthedocs.io/en/latest/ - - -Tests ------ - -You can run tests from the project root after cloning with: - -.. code-block:: sh - - $ python setup.py test - - diff --git a/contrib/python/PyJWT/py2/.dist-info/entry_points.txt b/contrib/python/PyJWT/py2/.dist-info/entry_points.txt deleted file mode 100644 index 78717b26610..00000000000 --- a/contrib/python/PyJWT/py2/.dist-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -pyjwt = jwt.__main__:main - diff --git a/contrib/python/PyJWT/py2/.dist-info/top_level.txt b/contrib/python/PyJWT/py2/.dist-info/top_level.txt deleted file mode 100644 index 27ccc9bc3a9..00000000000 --- a/contrib/python/PyJWT/py2/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -jwt diff --git a/contrib/python/PyJWT/py2/jwt/__init__.py b/contrib/python/PyJWT/py2/jwt/__init__.py deleted file mode 100644 index 946983f0221..00000000000 --- a/contrib/python/PyJWT/py2/jwt/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa - -""" -JSON Web Token implementation - -Minimum implementation based on this spec: -http://self-issued.info/docs/draft-jones-json-web-token-01.html -""" - - -__title__ = 'pyjwt' -__version__ = '1.7.1' -__author__ = 'José Padilla' -__license__ = 'MIT' -__copyright__ = 'Copyright 2015-2018 José Padilla' - - -from .api_jwt import ( - encode, decode, register_algorithm, unregister_algorithm, - get_unverified_header, PyJWT -) -from .api_jws import PyJWS -from .exceptions import ( - InvalidTokenError, DecodeError, InvalidAlgorithmError, - InvalidAudienceError, ExpiredSignatureError, ImmatureSignatureError, - InvalidIssuedAtError, InvalidIssuerError, ExpiredSignature, - InvalidAudience, InvalidIssuer, MissingRequiredClaimError, - InvalidSignatureError, - PyJWTError, -) diff --git a/contrib/python/PyJWT/py2/jwt/__main__.py b/contrib/python/PyJWT/py2/jwt/__main__.py deleted file mode 100644 index bf50aabf4af..00000000000 --- a/contrib/python/PyJWT/py2/jwt/__main__.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python - -from __future__ import absolute_import, print_function - -import argparse -import json -import sys -import time - -from . import DecodeError, __version__, decode, encode - - -def encode_payload(args): - # Try to encode - if args.key is None: - raise ValueError('Key is required when encoding. See --help for usage.') - - # Build payload object to encode - payload = {} - - for arg in args.payload: - k, v = arg.split('=', 1) - - # exp +offset special case? - if k == 'exp' and v[0] == '+' and len(v) > 1: - v = str(int(time.time()+int(v[1:]))) - - # Cast to integer? - if v.isdigit(): - v = int(v) - else: - # Cast to float? - try: - v = float(v) - except ValueError: - pass - - # Cast to true, false, or null? - constants = {'true': True, 'false': False, 'null': None} - - if v in constants: - v = constants[v] - - payload[k] = v - - token = encode( - payload, - key=args.key, - algorithm=args.algorithm - ) - - return token.decode('utf-8') - - -def decode_payload(args): - try: - if args.token: - token = args.token - else: - if sys.stdin.isatty(): - token = sys.stdin.readline().strip() - else: - raise IOError('Cannot read from stdin: terminal not a TTY') - - token = token.encode('utf-8') - data = decode(token, key=args.key, verify=args.verify) - - return json.dumps(data) - - except DecodeError as e: - raise DecodeError('There was an error decoding the token: %s' % e) - - -def build_argparser(): - - usage = ''' - Encodes or decodes JSON Web Tokens based on input. - - %(prog)s [options] <command> [options] input - - Decoding examples: - - %(prog)s --key=secret decode json.web.token - %(prog)s decode --no-verify json.web.token - - Encoding requires the key option and takes space separated key/value pairs - separated by equals (=) as input. Examples: - - %(prog)s --key=secret encode iss=me exp=1302049071 - %(prog)s --key=secret encode foo=bar exp=+10 - - The exp key is special and can take an offset to current Unix time. - ''' - - arg_parser = argparse.ArgumentParser( - prog='pyjwt', - usage=usage - ) - - arg_parser.add_argument( - '-v', '--version', - action='version', - version='%(prog)s ' + __version__ - ) - - arg_parser.add_argument( - '--key', - dest='key', - metavar='KEY', - default=None, - help='set the secret key to sign with' - ) - - arg_parser.add_argument( - '--alg', - dest='algorithm', - metavar='ALG', - default='HS256', - help='set crypto algorithm to sign with. default=HS256' - ) - - subparsers = arg_parser.add_subparsers( - title='PyJWT subcommands', - description='valid subcommands', - help='additional help' - ) - - # Encode subcommand - encode_parser = subparsers.add_parser('encode', help='use to encode a supplied payload') - - payload_help = """Payload to encode. Must be a space separated list of key/value - pairs separated by equals (=) sign.""" - - encode_parser.add_argument('payload', nargs='+', help=payload_help) - encode_parser.set_defaults(func=encode_payload) - - # Decode subcommand - decode_parser = subparsers.add_parser('decode', help='use to decode a supplied JSON web token') - decode_parser.add_argument( - 'token', - help='JSON web token to decode.', - nargs='?') - - decode_parser.add_argument( - '-n', '--no-verify', - action='store_false', - dest='verify', - default=True, - help='ignore signature and claims verification on decode' - ) - - decode_parser.set_defaults(func=decode_payload) - - return arg_parser - - -def main(): - arg_parser = build_argparser() - - try: - arguments = arg_parser.parse_args(sys.argv[1:]) - - output = arguments.func(arguments) - - print(output) - except Exception as e: - print('There was an unforseen error: ', e) - arg_parser.print_help() diff --git a/contrib/python/PyJWT/py2/jwt/algorithms.py b/contrib/python/PyJWT/py2/jwt/algorithms.py deleted file mode 100644 index 13436883417..00000000000 --- a/contrib/python/PyJWT/py2/jwt/algorithms.py +++ /dev/null @@ -1,403 +0,0 @@ -import hashlib -import hmac -import json - - -from .compat import constant_time_compare, string_types -from .exceptions import InvalidKeyError -from .utils import ( - base64url_decode, base64url_encode, der_to_raw_signature, - force_bytes, force_unicode, from_base64url_uint, raw_to_der_signature, - to_base64url_uint -) - -try: - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.serialization import ( - load_pem_private_key, load_pem_public_key, load_ssh_public_key - ) - from cryptography.hazmat.primitives.asymmetric.rsa import ( - RSAPrivateKey, RSAPublicKey, RSAPrivateNumbers, RSAPublicNumbers, - rsa_recover_prime_factors, rsa_crt_dmp1, rsa_crt_dmq1, rsa_crt_iqmp - ) - from cryptography.hazmat.primitives.asymmetric.ec import ( - EllipticCurvePrivateKey, EllipticCurvePublicKey - ) - from cryptography.hazmat.primitives.asymmetric import ec, padding - from cryptography.hazmat.backends import default_backend - from cryptography.exceptions import InvalidSignature - - has_crypto = True -except ImportError: - has_crypto = False - -requires_cryptography = set(['RS256', 'RS384', 'RS512', 'ES256', 'ES384', - 'ES521', 'ES512', 'PS256', 'PS384', 'PS512']) - - -def get_default_algorithms(): - """ - Returns the algorithms that are implemented by the library. - """ - default_algorithms = { - 'none': NoneAlgorithm(), - 'HS256': HMACAlgorithm(HMACAlgorithm.SHA256), - 'HS384': HMACAlgorithm(HMACAlgorithm.SHA384), - 'HS512': HMACAlgorithm(HMACAlgorithm.SHA512) - } - - if has_crypto: - default_algorithms.update({ - 'RS256': RSAAlgorithm(RSAAlgorithm.SHA256), - 'RS384': RSAAlgorithm(RSAAlgorithm.SHA384), - 'RS512': RSAAlgorithm(RSAAlgorithm.SHA512), - 'ES256': ECAlgorithm(ECAlgorithm.SHA256), - 'ES384': ECAlgorithm(ECAlgorithm.SHA384), - 'ES521': ECAlgorithm(ECAlgorithm.SHA512), - 'ES512': ECAlgorithm(ECAlgorithm.SHA512), # Backward compat for #219 fix - 'PS256': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA256), - 'PS384': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA384), - 'PS512': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA512) - }) - - return default_algorithms - - -class Algorithm(object): - """ - The interface for an algorithm used to sign and verify tokens. - """ - def prepare_key(self, key): - """ - Performs necessary validation and conversions on the key and returns - the key value in the proper format for sign() and verify(). - """ - raise NotImplementedError - - def sign(self, msg, key): - """ - Returns a digital signature for the specified message - using the specified key value. - """ - raise NotImplementedError - - def verify(self, msg, key, sig): - """ - Verifies that the specified digital signature is valid - for the specified message and key values. - """ - raise NotImplementedError - - @staticmethod - def to_jwk(key_obj): - """ - Serializes a given RSA key into a JWK - """ - raise NotImplementedError - - @staticmethod - def from_jwk(jwk): - """ - Deserializes a given RSA key from JWK back into a PublicKey or PrivateKey object - """ - raise NotImplementedError - - -class NoneAlgorithm(Algorithm): - """ - Placeholder for use when no signing or verification - operations are required. - """ - def prepare_key(self, key): - if key == '': - key = None - - if key is not None: - raise InvalidKeyError('When alg = "none", key value must be None.') - - return key - - def sign(self, msg, key): - return b'' - - def verify(self, msg, key, sig): - return False - - -class HMACAlgorithm(Algorithm): - """ - Performs signing and verification operations using HMAC - and the specified hash function. - """ - SHA256 = hashlib.sha256 - SHA384 = hashlib.sha384 - SHA512 = hashlib.sha512 - - def __init__(self, hash_alg): - self.hash_alg = hash_alg - - def prepare_key(self, key): - key = force_bytes(key) - - invalid_strings = [ - b'-----BEGIN PUBLIC KEY-----', - b'-----BEGIN CERTIFICATE-----', - b'-----BEGIN RSA PUBLIC KEY-----', - b'ssh-rsa' - ] - - if any([string_value in key for string_value in invalid_strings]): - raise InvalidKeyError( - 'The specified key is an asymmetric key or x509 certificate and' - ' should not be used as an HMAC secret.') - - return key - - @staticmethod - def to_jwk(key_obj): - return json.dumps({ - 'k': force_unicode(base64url_encode(force_bytes(key_obj))), - 'kty': 'oct' - }) - - @staticmethod - def from_jwk(jwk): - obj = json.loads(jwk) - - if obj.get('kty') != 'oct': - raise InvalidKeyError('Not an HMAC key') - - return base64url_decode(obj['k']) - - def sign(self, msg, key): - return hmac.new(key, msg, self.hash_alg).digest() - - def verify(self, msg, key, sig): - return constant_time_compare(sig, self.sign(msg, key)) - - -if has_crypto: - - class RSAAlgorithm(Algorithm): - """ - Performs signing and verification operations using - RSASSA-PKCS-v1_5 and the specified hash function. - """ - SHA256 = hashes.SHA256 - SHA384 = hashes.SHA384 - SHA512 = hashes.SHA512 - - def __init__(self, hash_alg): - self.hash_alg = hash_alg - - def prepare_key(self, key): - if isinstance(key, RSAPrivateKey) or \ - isinstance(key, RSAPublicKey): - return key - - if isinstance(key, string_types): - key = force_bytes(key) - - try: - if key.startswith(b'ssh-rsa'): - key = load_ssh_public_key(key, backend=default_backend()) - else: - key = load_pem_private_key(key, password=None, backend=default_backend()) - except ValueError: - key = load_pem_public_key(key, backend=default_backend()) - else: - raise TypeError('Expecting a PEM-formatted key.') - - return key - - @staticmethod - def to_jwk(key_obj): - obj = None - - if getattr(key_obj, 'private_numbers', None): - # Private key - numbers = key_obj.private_numbers() - - obj = { - 'kty': 'RSA', - 'key_ops': ['sign'], - 'n': force_unicode(to_base64url_uint(numbers.public_numbers.n)), - 'e': force_unicode(to_base64url_uint(numbers.public_numbers.e)), - 'd': force_unicode(to_base64url_uint(numbers.d)), - 'p': force_unicode(to_base64url_uint(numbers.p)), - 'q': force_unicode(to_base64url_uint(numbers.q)), - 'dp': force_unicode(to_base64url_uint(numbers.dmp1)), - 'dq': force_unicode(to_base64url_uint(numbers.dmq1)), - 'qi': force_unicode(to_base64url_uint(numbers.iqmp)) - } - - elif getattr(key_obj, 'verify', None): - # Public key - numbers = key_obj.public_numbers() - - obj = { - 'kty': 'RSA', - 'key_ops': ['verify'], - 'n': force_unicode(to_base64url_uint(numbers.n)), - 'e': force_unicode(to_base64url_uint(numbers.e)) - } - else: - raise InvalidKeyError('Not a public or private key') - - return json.dumps(obj) - - @staticmethod - def from_jwk(jwk): - try: - obj = json.loads(jwk) - except ValueError: - raise InvalidKeyError('Key is not valid JSON') - - if obj.get('kty') != 'RSA': - raise InvalidKeyError('Not an RSA key') - - if 'd' in obj and 'e' in obj and 'n' in obj: - # Private key - if 'oth' in obj: - raise InvalidKeyError('Unsupported RSA private key: > 2 primes not supported') - - other_props = ['p', 'q', 'dp', 'dq', 'qi'] - props_found = [prop in obj for prop in other_props] - any_props_found = any(props_found) - - if any_props_found and not all(props_found): - raise InvalidKeyError('RSA key must include all parameters if any are present besides d') - - public_numbers = RSAPublicNumbers( - from_base64url_uint(obj['e']), from_base64url_uint(obj['n']) - ) - - if any_props_found: - numbers = RSAPrivateNumbers( - d=from_base64url_uint(obj['d']), - p=from_base64url_uint(obj['p']), - q=from_base64url_uint(obj['q']), - dmp1=from_base64url_uint(obj['dp']), - dmq1=from_base64url_uint(obj['dq']), - iqmp=from_base64url_uint(obj['qi']), - public_numbers=public_numbers - ) - else: - d = from_base64url_uint(obj['d']) - p, q = rsa_recover_prime_factors( - public_numbers.n, d, public_numbers.e - ) - - numbers = RSAPrivateNumbers( - d=d, - p=p, - q=q, - dmp1=rsa_crt_dmp1(d, p), - dmq1=rsa_crt_dmq1(d, q), - iqmp=rsa_crt_iqmp(p, q), - public_numbers=public_numbers - ) - - return numbers.private_key(default_backend()) - elif 'n' in obj and 'e' in obj: - # Public key - numbers = RSAPublicNumbers( - from_base64url_uint(obj['e']), from_base64url_uint(obj['n']) - ) - - return numbers.public_key(default_backend()) - else: - raise InvalidKeyError('Not a public or private key') - - def sign(self, msg, key): - return key.sign(msg, padding.PKCS1v15(), self.hash_alg()) - - def verify(self, msg, key, sig): - try: - key.verify(sig, msg, padding.PKCS1v15(), self.hash_alg()) - return True - except InvalidSignature: - return False - - class ECAlgorithm(Algorithm): - """ - Performs signing and verification operations using - ECDSA and the specified hash function - """ - SHA256 = hashes.SHA256 - SHA384 = hashes.SHA384 - SHA512 = hashes.SHA512 - - def __init__(self, hash_alg): - self.hash_alg = hash_alg - - def prepare_key(self, key): - if isinstance(key, EllipticCurvePrivateKey) or \ - isinstance(key, EllipticCurvePublicKey): - return key - - if isinstance(key, string_types): - key = force_bytes(key) - - # Attempt to load key. We don't know if it's - # a Signing Key or a Verifying Key, so we try - # the Verifying Key first. - try: - if key.startswith(b'ecdsa-sha2-'): - key = load_ssh_public_key(key, backend=default_backend()) - else: - key = load_pem_public_key(key, backend=default_backend()) - except ValueError: - key = load_pem_private_key(key, password=None, backend=default_backend()) - - else: - raise TypeError('Expecting a PEM-formatted key.') - - return key - - def sign(self, msg, key): - der_sig = key.sign(msg, ec.ECDSA(self.hash_alg())) - - return der_to_raw_signature(der_sig, key.curve) - - def verify(self, msg, key, sig): - try: - der_sig = raw_to_der_signature(sig, key.curve) - except ValueError: - return False - - try: - key.verify(der_sig, msg, ec.ECDSA(self.hash_alg())) - return True - except InvalidSignature: - return False - - class RSAPSSAlgorithm(RSAAlgorithm): - """ - Performs a signature using RSASSA-PSS with MGF1 - """ - - def sign(self, msg, key): - return key.sign( - msg, - padding.PSS( - mgf=padding.MGF1(self.hash_alg()), - salt_length=self.hash_alg.digest_size - ), - self.hash_alg() - ) - - def verify(self, msg, key, sig): - try: - key.verify( - sig, - msg, - padding.PSS( - mgf=padding.MGF1(self.hash_alg()), - salt_length=self.hash_alg.digest_size - ), - self.hash_alg() - ) - return True - except InvalidSignature: - return False diff --git a/contrib/python/PyJWT/py2/jwt/api_jws.py b/contrib/python/PyJWT/py2/jwt/api_jws.py deleted file mode 100644 index a9354adb06c..00000000000 --- a/contrib/python/PyJWT/py2/jwt/api_jws.py +++ /dev/null @@ -1,242 +0,0 @@ -import binascii -import json -import warnings -try: - # import required by mypy to perform type checking, not used for normal execution - from typing import Callable, Dict, List, Optional, Union # NOQA -except ImportError: - pass - -from .algorithms import ( - Algorithm, get_default_algorithms, has_crypto, requires_cryptography # NOQA -) -from .compat import Mapping, binary_type, string_types, text_type -from .exceptions import ( - DecodeError, InvalidAlgorithmError, InvalidSignatureError, - InvalidTokenError -) -from .utils import base64url_decode, base64url_encode, force_bytes, merge_dict - - -class PyJWS(object): - header_typ = 'JWT' - - def __init__(self, algorithms=None, options=None): - self._algorithms = get_default_algorithms() - self._valid_algs = (set(algorithms) if algorithms is not None - else set(self._algorithms)) - - # Remove algorithms that aren't on the whitelist - for key in list(self._algorithms.keys()): - if key not in self._valid_algs: - del self._algorithms[key] - - if not options: - options = {} - - self.options = merge_dict(self._get_default_options(), options) - - @staticmethod - def _get_default_options(): - return { - 'verify_signature': True - } - - def register_algorithm(self, alg_id, alg_obj): - """ - Registers a new Algorithm for use when creating and verifying tokens. - """ - if alg_id in self._algorithms: - raise ValueError('Algorithm already has a handler.') - - if not isinstance(alg_obj, Algorithm): - raise TypeError('Object is not of type `Algorithm`') - - self._algorithms[alg_id] = alg_obj - self._valid_algs.add(alg_id) - - def unregister_algorithm(self, alg_id): - """ - Unregisters an Algorithm for use when creating and verifying tokens - Throws KeyError if algorithm is not registered. - """ - if alg_id not in self._algorithms: - raise KeyError('The specified algorithm could not be removed' - ' because it is not registered.') - - del self._algorithms[alg_id] - self._valid_algs.remove(alg_id) - - def get_algorithms(self): - """ - Returns a list of supported values for the 'alg' parameter. - """ - return list(self._valid_algs) - - def encode(self, - payload, # type: Union[Dict, bytes] - key, # type: str - algorithm='HS256', # type: str - headers=None, # type: Optional[Dict] - json_encoder=None # type: Optional[Callable] - ): - segments = [] - - if algorithm is None: - algorithm = 'none' - - if algorithm not in self._valid_algs: - pass - - # Header - header = {'typ': self.header_typ, 'alg': algorithm} - - if headers: - self._validate_headers(headers) - header.update(headers) - - json_header = force_bytes( - json.dumps( - header, - separators=(',', ':'), - cls=json_encoder - ) - ) - - segments.append(base64url_encode(json_header)) - segments.append(base64url_encode(payload)) - - # Segments - signing_input = b'.'.join(segments) - try: - alg_obj = self._algorithms[algorithm] - key = alg_obj.prepare_key(key) - signature = alg_obj.sign(signing_input, key) - - except KeyError: - if not has_crypto and algorithm in requires_cryptography: - raise NotImplementedError( - "Algorithm '%s' could not be found. Do you have cryptography " - "installed?" % algorithm - ) - else: - raise NotImplementedError('Algorithm not supported') - - segments.append(base64url_encode(signature)) - - return b'.'.join(segments) - - def decode(self, - jwt, # type: str - key='', # type: str - verify=True, # type: bool - algorithms=None, # type: List[str] - options=None, # type: Dict - **kwargs): - - merged_options = merge_dict(self.options, options) - verify_signature = merged_options['verify_signature'] - - if verify_signature and not algorithms: - warnings.warn( - 'It is strongly recommended that you pass in a ' + - 'value for the "algorithms" argument when calling decode(). ' + - 'This argument will be mandatory in a future version.', - DeprecationWarning - ) - - payload, signing_input, header, signature = self._load(jwt) - - if not verify: - warnings.warn('The verify parameter is deprecated. ' - 'Please use verify_signature in options instead.', - DeprecationWarning, stacklevel=2) - elif verify_signature: - self._verify_signature(payload, signing_input, header, signature, - key, algorithms) - - return payload - - def get_unverified_header(self, jwt): - """Returns back the JWT header parameters as a dict() - - Note: The signature is not verified so the header parameters - should not be fully trusted until signature verification is complete - """ - headers = self._load(jwt)[2] - self._validate_headers(headers) - - return headers - - def _load(self, jwt): - if isinstance(jwt, text_type): - jwt = jwt.encode('utf-8') - - if not issubclass(type(jwt), binary_type): - raise DecodeError("Invalid token type. Token must be a {0}".format( - binary_type)) - - try: - signing_input, crypto_segment = jwt.rsplit(b'.', 1) - header_segment, payload_segment = signing_input.split(b'.', 1) - except ValueError: - raise DecodeError('Not enough segments') - - try: - header_data = base64url_decode(header_segment) - except (TypeError, binascii.Error): - raise DecodeError('Invalid header padding') - - try: - header = json.loads(header_data.decode('utf-8')) - except ValueError as e: - raise DecodeError('Invalid header string: %s' % e) - - if not isinstance(header, Mapping): - raise DecodeError('Invalid header string: must be a json object') - - try: - payload = base64url_decode(payload_segment) - except (TypeError, binascii.Error): - raise DecodeError('Invalid payload padding') - - try: - signature = base64url_decode(crypto_segment) - except (TypeError, binascii.Error): - raise DecodeError('Invalid crypto padding') - - return (payload, signing_input, header, signature) - - def _verify_signature(self, payload, signing_input, header, signature, - key='', algorithms=None): - - alg = header.get('alg') - - if algorithms is not None and alg not in algorithms: - raise InvalidAlgorithmError('The specified alg value is not allowed') - - try: - alg_obj = self._algorithms[alg] - key = alg_obj.prepare_key(key) - - if not alg_obj.verify(signing_input, key, signature): - raise InvalidSignatureError('Signature verification failed') - - except KeyError: - raise InvalidAlgorithmError('Algorithm not supported') - - def _validate_headers(self, headers): - if 'kid' in headers: - self._validate_kid(headers['kid']) - - def _validate_kid(self, kid): - if not isinstance(kid, string_types): - raise InvalidTokenError('Key ID header parameter must be a string') - - -_jws_global_obj = PyJWS() -encode = _jws_global_obj.encode -decode = _jws_global_obj.decode -register_algorithm = _jws_global_obj.register_algorithm -unregister_algorithm = _jws_global_obj.unregister_algorithm -get_unverified_header = _jws_global_obj.get_unverified_header diff --git a/contrib/python/PyJWT/py2/jwt/api_jwt.py b/contrib/python/PyJWT/py2/jwt/api_jwt.py deleted file mode 100644 index 85504acf930..00000000000 --- a/contrib/python/PyJWT/py2/jwt/api_jwt.py +++ /dev/null @@ -1,222 +0,0 @@ -import json -import warnings -from calendar import timegm -from datetime import datetime, timedelta -try: - # import required by mypy to perform type checking, not used for normal execution - from typing import Callable, Dict, List, Optional, Union # NOQA -except ImportError: - pass - -from .api_jws import PyJWS -from .algorithms import Algorithm, get_default_algorithms # NOQA -from .compat import Iterable, Mapping, string_types -from .exceptions import ( - DecodeError, ExpiredSignatureError, ImmatureSignatureError, - InvalidAudienceError, InvalidIssuedAtError, - InvalidIssuerError, MissingRequiredClaimError -) -from .utils import merge_dict - - -class PyJWT(PyJWS): - header_type = 'JWT' - - @staticmethod - def _get_default_options(): - # type: () -> Dict[str, bool] - return { - 'verify_signature': True, - 'verify_exp': True, - 'verify_nbf': True, - 'verify_iat': True, - 'verify_aud': True, - 'verify_iss': True, - 'require_exp': False, - 'require_iat': False, - 'require_nbf': False - } - - def encode(self, - payload, # type: Union[Dict, bytes] - key, # type: str - algorithm='HS256', # type: str - headers=None, # type: Optional[Dict] - json_encoder=None # type: Optional[Callable] - ): - # Check that we get a mapping - if not isinstance(payload, Mapping): - raise TypeError('Expecting a mapping object, as JWT only supports ' - 'JSON objects as payloads.') - - # Payload - for time_claim in ['exp', 'iat', 'nbf']: - # Convert datetime to a intDate value in known time-format claims - if isinstance(payload.get(time_claim), datetime): - payload[time_claim] = timegm(payload[time_claim].utctimetuple()) # type: ignore - - json_payload = json.dumps( - payload, - separators=(',', ':'), - cls=json_encoder - ).encode('utf-8') - - return super(PyJWT, self).encode( - json_payload, key, algorithm, headers, json_encoder - ) - - def decode(self, - jwt, # type: str - key='', # type: str - verify=True, # type: bool - algorithms=None, # type: List[str] - options=None, # type: Dict - **kwargs): - - if verify and not algorithms: - warnings.warn( - 'It is strongly recommended that you pass in a ' + - 'value for the "algorithms" argument when calling decode(). ' + - 'This argument will be mandatory in a future version.', - DeprecationWarning - ) - - payload, _, _, _ = self._load(jwt) - - if options is None: - options = {'verify_signature': verify} - else: - options.setdefault('verify_signature', verify) - - decoded = super(PyJWT, self).decode( - jwt, key=key, algorithms=algorithms, options=options, **kwargs - ) - - try: - payload = json.loads(decoded.decode('utf-8')) - except ValueError as e: - raise DecodeError('Invalid payload string: %s' % e) - if not isinstance(payload, Mapping): - raise DecodeError('Invalid payload string: must be a json object') - - if verify: - merged_options = merge_dict(self.options, options) - self._validate_claims(payload, merged_options, **kwargs) - - return payload - - def _validate_claims(self, payload, options, audience=None, issuer=None, - leeway=0, **kwargs): - - if 'verify_expiration' in kwargs: - options['verify_exp'] = kwargs.get('verify_expiration', True) - warnings.warn('The verify_expiration parameter is deprecated. ' - 'Please use verify_exp in options instead.', - DeprecationWarning) - - if isinstance(leeway, timedelta): - leeway = leeway.total_seconds() - - if not isinstance(audience, (string_types, type(None), Iterable)): - raise TypeError('audience must be a string, iterable, or None') - - self._validate_required_claims(payload, options) - - now = timegm(datetime.utcnow().utctimetuple()) - - if 'iat' in payload and options.get('verify_iat'): - self._validate_iat(payload, now, leeway) - - if 'nbf' in payload and options.get('verify_nbf'): - self._validate_nbf(payload, now, leeway) - - if 'exp' in payload and options.get('verify_exp'): - self._validate_exp(payload, now, leeway) - - if options.get('verify_iss'): - self._validate_iss(payload, issuer) - - if options.get('verify_aud'): - self._validate_aud(payload, audience) - - def _validate_required_claims(self, payload, options): - if options.get('require_exp') and payload.get('exp') is None: - raise MissingRequiredClaimError('exp') - - if options.get('require_iat') and payload.get('iat') is None: - raise MissingRequiredClaimError('iat') - - if options.get('require_nbf') and payload.get('nbf') is None: - raise MissingRequiredClaimError('nbf') - - def _validate_iat(self, payload, now, leeway): - try: - int(payload['iat']) - except ValueError: - raise InvalidIssuedAtError('Issued At claim (iat) must be an integer.') - - def _validate_nbf(self, payload, now, leeway): - try: - nbf = int(payload['nbf']) - except ValueError: - raise DecodeError('Not Before claim (nbf) must be an integer.') - - if nbf > (now + leeway): - raise ImmatureSignatureError('The token is not yet valid (nbf)') - - def _validate_exp(self, payload, now, leeway): - try: - exp = int(payload['exp']) - except ValueError: - raise DecodeError('Expiration Time claim (exp) must be an' - ' integer.') - - if exp < (now - leeway): - raise ExpiredSignatureError('Signature has expired') - - def _validate_aud(self, payload, audience): - if audience is None and 'aud' not in payload: - return - - if audience is not None and 'aud' not in payload: - # Application specified an audience, but it could not be - # verified since the token does not contain a claim. - raise MissingRequiredClaimError('aud') - - if audience is None and 'aud' in payload: - # Application did not specify an audience, but - # the token has the 'aud' claim - raise InvalidAudienceError('Invalid audience') - - audience_claims = payload['aud'] - - if isinstance(audience_claims, string_types): - audience_claims = [audience_claims] - if not isinstance(audience_claims, list): - raise InvalidAudienceError('Invalid claim format in token') - if any(not isinstance(c, string_types) for c in audience_claims): - raise InvalidAudienceError('Invalid claim format in token') - - if isinstance(audience, string_types): - audience = [audience] - - if not any(aud in audience_claims for aud in audience): - raise InvalidAudienceError('Invalid audience') - - def _validate_iss(self, payload, issuer): - if issuer is None: - return - - if 'iss' not in payload: - raise MissingRequiredClaimError('iss') - - if payload['iss'] != issuer: - raise InvalidIssuerError('Invalid issuer') - - -_jwt_global_obj = PyJWT() -encode = _jwt_global_obj.encode -decode = _jwt_global_obj.decode -register_algorithm = _jwt_global_obj.register_algorithm -unregister_algorithm = _jwt_global_obj.unregister_algorithm -get_unverified_header = _jwt_global_obj.get_unverified_header diff --git a/contrib/python/PyJWT/py2/jwt/compat.py b/contrib/python/PyJWT/py2/jwt/compat.py deleted file mode 100644 index e79e258e56d..00000000000 --- a/contrib/python/PyJWT/py2/jwt/compat.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -The `compat` module provides support for backwards compatibility with older -versions of python, and compatibility wrappers around optional packages. -""" -# flake8: noqa -import hmac -import struct -import sys - - -PY3 = sys.version_info[0] == 3 - - -if PY3: - text_type = str - binary_type = bytes -else: - text_type = unicode - binary_type = str - -string_types = (text_type, binary_type) - -try: - # Importing ABCs from collections will be removed in PY3.8 - from collections.abc import Iterable, Mapping -except ImportError: - from collections import Iterable, Mapping - -try: - constant_time_compare = hmac.compare_digest -except AttributeError: - # Fallback for Python < 2.7 - def constant_time_compare(val1, val2): - """ - Returns True if the two strings are equal, False otherwise. - - The time taken is independent of the number of characters that match. - """ - if len(val1) != len(val2): - return False - - result = 0 - - for x, y in zip(val1, val2): - result |= ord(x) ^ ord(y) - - return result == 0 - -# Use int.to_bytes if it exists (Python 3) -if getattr(int, 'to_bytes', None): - def bytes_from_int(val): - remaining = val - byte_length = 0 - - while remaining != 0: - remaining = remaining >> 8 - byte_length += 1 - - return val.to_bytes(byte_length, 'big', signed=False) -else: - def bytes_from_int(val): - buf = [] - while val: - val, remainder = divmod(val, 256) - buf.append(remainder) - - buf.reverse() - return struct.pack('%sB' % len(buf), *buf) diff --git a/contrib/python/PyJWT/py2/jwt/contrib/__init__.py b/contrib/python/PyJWT/py2/jwt/contrib/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/PyJWT/py2/jwt/contrib/__init__.py +++ /dev/null diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py +++ /dev/null diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py deleted file mode 100644 index bf0dea5ae28..00000000000 --- a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py +++ /dev/null @@ -1,60 +0,0 @@ -# Note: This file is named py_ecdsa.py because import behavior in Python 2 -# would cause ecdsa.py to squash the ecdsa library that it depends upon. - -import hashlib - -import ecdsa - -from jwt.algorithms import Algorithm -from jwt.compat import string_types, text_type - - -class ECAlgorithm(Algorithm): - """ - Performs signing and verification operations using - ECDSA and the specified hash function - - This class requires the ecdsa package to be installed. - - This is based off of the implementation in PyJWT 0.3.2 - """ - SHA256 = hashlib.sha256 - SHA384 = hashlib.sha384 - SHA512 = hashlib.sha512 - - def __init__(self, hash_alg): - self.hash_alg = hash_alg - - def prepare_key(self, key): - - if isinstance(key, ecdsa.SigningKey) or \ - isinstance(key, ecdsa.VerifyingKey): - return key - - if isinstance(key, string_types): - if isinstance(key, text_type): - key = key.encode('utf-8') - - # Attempt to load key. We don't know if it's - # a Signing Key or a Verifying Key, so we try - # the Verifying Key first. - try: - key = ecdsa.VerifyingKey.from_pem(key) - except ecdsa.der.UnexpectedDER: - key = ecdsa.SigningKey.from_pem(key) - - else: - raise TypeError('Expecting a PEM-formatted key.') - - return key - - def sign(self, msg, key): - return key.sign(msg, hashfunc=self.hash_alg, - sigencode=ecdsa.util.sigencode_string) - - def verify(self, msg, key, sig): - try: - return key.verify(sig, msg, hashfunc=self.hash_alg, - sigdecode=ecdsa.util.sigdecode_string) - except AssertionError: - return False diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py deleted file mode 100644 index e49cdbfe40f..00000000000 --- a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py +++ /dev/null @@ -1,46 +0,0 @@ -import Crypto.Hash.SHA256 -import Crypto.Hash.SHA384 -import Crypto.Hash.SHA512 -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_v1_5 - -from jwt.algorithms import Algorithm -from jwt.compat import string_types, text_type - - -class RSAAlgorithm(Algorithm): - """ - Performs signing and verification operations using - RSASSA-PKCS-v1_5 and the specified hash function. - - This class requires PyCrypto package to be installed. - - This is based off of the implementation in PyJWT 0.3.2 - """ - SHA256 = Crypto.Hash.SHA256 - SHA384 = Crypto.Hash.SHA384 - SHA512 = Crypto.Hash.SHA512 - - def __init__(self, hash_alg): - self.hash_alg = hash_alg - - def prepare_key(self, key): - - if isinstance(key, RSA._RSAobj): - return key - - if isinstance(key, string_types): - if isinstance(key, text_type): - key = key.encode('utf-8') - - key = RSA.importKey(key) - else: - raise TypeError('Expecting a PEM- or RSA-formatted key.') - - return key - - def sign(self, msg, key): - return PKCS1_v1_5.new(key).sign(self.hash_alg.new(msg)) - - def verify(self, msg, key, sig): - return PKCS1_v1_5.new(key).verify(self.hash_alg.new(msg), sig) diff --git a/contrib/python/PyJWT/py2/jwt/exceptions.py b/contrib/python/PyJWT/py2/jwt/exceptions.py deleted file mode 100644 index 2a6aa596ba0..00000000000 --- a/contrib/python/PyJWT/py2/jwt/exceptions.py +++ /dev/null @@ -1,59 +0,0 @@ -class PyJWTError(Exception): - """ - Base class for all exceptions - """ - pass - - -class InvalidTokenError(PyJWTError): - pass - - -class DecodeError(InvalidTokenError): - pass - - -class InvalidSignatureError(DecodeError): - pass - - -class ExpiredSignatureError(InvalidTokenError): - pass - - -class InvalidAudienceError(InvalidTokenError): - pass - - -class InvalidIssuerError(InvalidTokenError): - pass - - -class InvalidIssuedAtError(InvalidTokenError): - pass - - -class ImmatureSignatureError(InvalidTokenError): - pass - - -class InvalidKeyError(PyJWTError): - pass - - -class InvalidAlgorithmError(InvalidTokenError): - pass - - -class MissingRequiredClaimError(InvalidTokenError): - def __init__(self, claim): - self.claim = claim - - def __str__(self): - return 'Token is missing the "%s" claim' % self.claim - - -# Compatibility aliases (deprecated) -ExpiredSignature = ExpiredSignatureError -InvalidAudience = InvalidAudienceError -InvalidIssuer = InvalidIssuerError diff --git a/contrib/python/PyJWT/py2/jwt/help.py b/contrib/python/PyJWT/py2/jwt/help.py deleted file mode 100644 index 55e39ebb271..00000000000 --- a/contrib/python/PyJWT/py2/jwt/help.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import print_function - -import json -import platform -import sys - -from . import __version__ as pyjwt_version - -try: - import cryptography -except ImportError: - cryptography = None - -try: - import ecdsa -except ImportError: - ecdsa = None - - -def info(): - """ - Generate information for a bug report. - Based on the requests package help utility module. - """ - try: - platform_info = {"system": platform.system(), "release": platform.release()} - except IOError: - platform_info = {"system": "Unknown", "release": "Unknown"} - - implementation = platform.python_implementation() - - if implementation == "CPython": - implementation_version = platform.python_version() - elif implementation == "PyPy": - implementation_version = "%s.%s.%s" % ( - sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro, - ) - if sys.pypy_version_info.releaselevel != "final": - implementation_version = "".join( - [implementation_version, sys.pypy_version_info.releaselevel] - ) - else: - implementation_version = "Unknown" - - return { - "platform": platform_info, - "implementation": {"name": implementation, "version": implementation_version}, - "cryptography": {"version": getattr(cryptography, "__version__", "")}, - "pyjwt": {"version": pyjwt_version}, - } - - -def main(): - """Pretty-print the bug information as JSON.""" - print(json.dumps(info(), sort_keys=True, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/contrib/python/PyJWT/py2/jwt/utils.py b/contrib/python/PyJWT/py2/jwt/utils.py deleted file mode 100644 index b33c7a2d455..00000000000 --- a/contrib/python/PyJWT/py2/jwt/utils.py +++ /dev/null @@ -1,113 +0,0 @@ -import base64 -import binascii -import struct - -from .compat import binary_type, bytes_from_int, text_type - -try: - from cryptography.hazmat.primitives.asymmetric.utils import ( - decode_dss_signature, encode_dss_signature - ) -except ImportError: - pass - - -def force_unicode(value): - if isinstance(value, binary_type): - return value.decode('utf-8') - elif isinstance(value, text_type): - return value - else: - raise TypeError('Expected a string value') - - -def force_bytes(value): - if isinstance(value, text_type): - return value.encode('utf-8') - elif isinstance(value, binary_type): - return value - else: - raise TypeError('Expected a string value') - - -def base64url_decode(input): - if isinstance(input, text_type): - input = input.encode('ascii') - - rem = len(input) % 4 - - if rem > 0: - input += b'=' * (4 - rem) - - return base64.urlsafe_b64decode(input) - - -def base64url_encode(input): - return base64.urlsafe_b64encode(input).replace(b'=', b'') - - -def to_base64url_uint(val): - if val < 0: - raise ValueError('Must be a positive integer') - - int_bytes = bytes_from_int(val) - - if len(int_bytes) == 0: - int_bytes = b'\x00' - - return base64url_encode(int_bytes) - - -def from_base64url_uint(val): - if isinstance(val, text_type): - val = val.encode('ascii') - - data = base64url_decode(val) - - buf = struct.unpack('%sB' % len(data), data) - return int(''.join(["%02x" % byte for byte in buf]), 16) - - -def merge_dict(original, updates): - if not updates: - return original - - try: - merged_options = original.copy() - merged_options.update(updates) - except (AttributeError, ValueError) as e: - raise TypeError('original and updates must be a dictionary: %s' % e) - - return merged_options - - -def number_to_bytes(num, num_bytes): - padded_hex = '%0*x' % (2 * num_bytes, num) - big_endian = binascii.a2b_hex(padded_hex.encode('ascii')) - return big_endian - - -def bytes_to_number(string): - return int(binascii.b2a_hex(string), 16) - - -def der_to_raw_signature(der_sig, curve): - num_bits = curve.key_size - num_bytes = (num_bits + 7) // 8 - - r, s = decode_dss_signature(der_sig) - - return number_to_bytes(r, num_bytes) + number_to_bytes(s, num_bytes) - - -def raw_to_der_signature(raw_sig, curve): - num_bits = curve.key_size - num_bytes = (num_bits + 7) // 8 - - if len(raw_sig) != 2 * num_bytes: - raise ValueError('Invalid signature') - - r = bytes_to_number(raw_sig[:num_bytes]) - s = bytes_to_number(raw_sig[num_bytes:]) - - return encode_dss_signature(r, s) diff --git a/contrib/python/PyJWT/py2/ya.make b/contrib/python/PyJWT/py2/ya.make deleted file mode 100644 index 57a9352fba1..00000000000 --- a/contrib/python/PyJWT/py2/ya.make +++ /dev/null @@ -1,43 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY2_LIBRARY() - -VERSION(1.7.1) - -LICENSE(MIT) - -PEERDIR( - contrib/python/cryptography -) - -NO_LINT() - -NO_CHECK_IMPORTS( - jwt.contrib.* -) - -PY_SRCS( - TOP_LEVEL - jwt/__init__.py - jwt/__main__.py - jwt/algorithms.py - jwt/api_jws.py - jwt/api_jwt.py - jwt/compat.py - jwt/contrib/__init__.py - jwt/contrib/algorithms/__init__.py - jwt/contrib/algorithms/py_ecdsa.py - jwt/contrib/algorithms/pycrypto.py - jwt/exceptions.py - jwt/help.py - jwt/utils.py -) - -RESOURCE_FILES( - PREFIX contrib/python/PyJWT/py2/ - .dist-info/METADATA - .dist-info/entry_points.txt - .dist-info/top_level.txt -) - -END() diff --git a/contrib/python/yandexcloud/.dist-info/METADATA b/contrib/python/yandexcloud/.dist-info/METADATA deleted file mode 100644 index dacbaf4ee30..00000000000 --- a/contrib/python/yandexcloud/.dist-info/METADATA +++ /dev/null @@ -1,203 +0,0 @@ -Metadata-Version: 2.1 -Name: yandexcloud -Version: 0.293.0 -Summary: The Yandex.Cloud official SDK -Home-page: https://github.com/yandex-cloud/python-sdk -Author: Yandex LLC -Author-email: [email protected] -License: MIT -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Description-Content-Type: text/markdown -License-File: LICENSE -License-File: AUTHORS -Requires-Dist: cryptography >=2.8 -Requires-Dist: grpcio >=1.64.0 -Requires-Dist: protobuf >=4.25.3 -Requires-Dist: googleapis-common-protos >=1.63.0 -Requires-Dist: pyjwt >=1.7.1 -Requires-Dist: requests >=2.22.0 -Requires-Dist: six >=1.14.0 - -[![PyPI Version][pypi-image]][pypi-url] -[![Build Status][build-image]][build-url] -[![License][license-image]][license-url] - -<!-- Badges --> - -[pypi-image]: https://img.shields.io/pypi/v/yandexcloud -[pypi-url]: https://pypi.org/project/yandexcloud/ -[build-image]: https://github.com/yandex-cloud/python-sdk/actions/workflows/run-tests.yml/badge.svg -[build-url]: https://github.com/yandex-cloud/python-sdk/actions/workflows/run-tests.yml -[license-image]: https://img.shields.io/github/license/yandex-cloud/python-sdk.svg -[license-url]: https://github.com/yandex-cloud/python-sdk/blob/master/LICENSE - -# Yandex.Cloud SDK (Python) - -Need to automate your infrastructure or use services provided by Yandex.Cloud? We've got you covered. - -Installation: - - pip install yandexcloud - -## Getting started - -There are several options for authorization your requests - OAuth Token, -Metadata Service (if you're executing your code inside VMs or Cloud Functions -running in Yandex.Cloud), Service Account Keys, and externally created IAM tokens. - -### OAuth Token - -```python -sdk = yandexcloud.SDK(token='AQAD-.....') -``` - -### Metadata Service - -Don't forget to assign Service Account for your Instance or Function and grant required roles. - -```python -sdk = yandexcloud.SDK() -``` - -### Service Account Keys - -```python -# you can store and read it from JSON file -sa_key = { - "id": "...", - "service_account_id": "...", - "private_key": "..." -} - -sdk = yandexcloud.SDK(service_account_key=sa_key) -``` - -### IAM tokens - -```python -sdk = yandexcloud.SDK(iam_token="t1.9eu...") -``` - -Check `examples` directory for more examples. - -### Override service endpoint - -#### Supported services - -| Service Name | Alias | -|------------------------------------------------------------------------|--------------------------| -| yandex.cloud.ai.foundation_models | ai-foundation-models | -| yandex.cloud.ai.llm | ai-llm | -| yandex.cloud.ai.stt | ai-stt | -| yandex.cloud.ai.translate | ai-translate | -| yandex.cloud.ai.tts | ai-speechkit | -| yandex.cloud.ai.vision | ai-vision | -| yandex.cloud.apploadbalancer | alb | -| yandex.cloud.billing | billing | -| yandex.cloud.cdn | cdn | -| yandex.cloud.certificatemanager.v1.certificate_content_service | certificate-manager-data | -| yandex.cloud.certificatemanager | certificate-manager | -| yandex.cloud.compute | compute | -| yandex.cloud.containerregistry | container-registry | -| yandex.cloud.dataproc.manager | dataproc-manager | -| yandex.cloud.dataproc | dataproc | -| yandex.cloud.datatransfer | datatransfer | -| yandex.cloud.dns | dns | -| yandex.cloud.endpoint | endpoint | -| yandex.cloud.iam | iam | -| yandex.cloud.iot.devices | iot-devices | -| yandex.cloud.k8s | managed-kubernetes | -| yandex.cloud.kms | kms | -| yandex.cloud.kms.v1.symmetric_crypto_service | kms-crypto | -| yandex.cloud.loadbalancer | load-balancer | -| yandex.cloud.loadtesting | loadtesting | -| yandex.cloud.lockbox.v1.payload_service | lockbox-payload | -| yandex.cloud.lockbox | lockbox | -| yandex.cloud.logging.v1.log_ingestion_service | log-ingestion | -| yandex.cloud.logging.v1.log_reading_service | log-reading | -| yandex.cloud.logging | logging | -| yandex.cloud.marketplace | marketplace | -| yandex.cloud.mdb.clickhouse | managed-clickhouse | -| yandex.cloud.mdb.elasticsearch | managed-elasticsearch | -| yandex.cloud.mdb.greenplum | managed-greenplum | -| yandex.cloud.mdb.kafka | managed-kafka | -| yandex.cloud.mdb.mongodb | managed-mongodb | -| yandex.cloud.mdb.mysql | managed-mysql | -| yandex.cloud.mdb.opensearch | managed-opensearch | -| yandex.cloud.mdb.postgresql | managed-postgresql | -| yandex.cloud.mdb.redis | managed-redis | -| yandex.cloud.mdb.sqlserver | managed-sqlserver | -| yandex.cloud.operation | operation | -| yandex.cloud.organizationmanager | organization-manager | -| yandex.cloud.resourcemanager | resource-manager | -| yandex.cloud.serverless.apigateway | serverless-apigateway | -| yandex.cloud.serverless.apigateway.websocket | apigateway-connections | -| yandex.cloud.serverless.containers | serverless-containers | -| yandex.cloud.serverless.functions | serverless-functions | -| yandex.cloud.serverless.triggers | serverless-triggers | -| yandex.cloud.storage | storage-api | -| yandex.cloud.vpc | vpc | -| yandex.cloud.ydb | ydb | - - -#### Override in client -```python -from yandex.cloud.vpc.v1.network_service_pb2_grpc import NetworkServiceStub -from yandexcloud import SDK - -sdk = SDK(iam_token="t1.9eu...") -new_network_client_endpoint = "example.new.vpc.very.new.yandex:50051" -insecure = False # by default is False, but if server does not support verification can be set to True -network_client = sdk.client(NetworkServiceStub, endpoint=new_network_client_endpoint, insecure=False) -``` - -#### Override in sdk config -To override endpoints provide dict in format {alias : new-endpoint} -```python -from yandex.cloud.vpc.v1.network_service_pb2_grpc import NetworkServiceStub -from yandexcloud import SDK -new_network_client_endpoint = "example.new.vpc.very.new.yandex:50051" -sdk = SDK(iam_token="t1.9eu...", endpoints={"vpc": new_network_client_endpoint}) -insecure = False # by default is False, but if server does not support verification can be set to True -network_client = sdk.client(NetworkServiceStub, insecure=False) -``` - -Notice: if both overrides are used for same endpoint, override by client has priority - -#### Switch SDK region -```python -from yandexcloud import SDK, set_up_yc_api_endpoint -kz_region_endpoint = "api.yandexcloud.kz" -# this will make SDK list endpoints from KZ yc installation -sdk = SDK(iam_token="t1.9eu...", endpoint="api.yandexcloud.kz") -# or you can use global function -set_up_yc_api_endpoint(kz_region_endpoint) -``` - -## Contributing -### Dependencies -Use `make deps` command to install library, its production and development dependencies. - -### Formatting -Use `make format` to autoformat code with black tool. - -### Tests -- `make test` to run tests for current python version -- `make lint` to run only linters for current python version -- `make tox-current` to run all checks (tests + code style checks + linters + format check) for current python version -- `make tox` to run all checks for all supported (installed in your system) python versions -- `make test-all-versions` to run all checks for all supported python versions in docker container - - -### Maintaining -If pull request consists of several meaningful commits, that should be preserved, -then use "Rebase and merge" option. Otherwise use "Squash and merge". - -New release (changelog, tag and pypi upload) will be automatically created -on each push to master via Github Actions workflow. diff --git a/contrib/python/yandexcloud/.dist-info/top_level.txt b/contrib/python/yandexcloud/.dist-info/top_level.txt deleted file mode 100644 index 3fbe85c20cb..00000000000 --- a/contrib/python/yandexcloud/.dist-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -yandex -yandexcloud diff --git a/contrib/python/yandexcloud/AUTHORS b/contrib/python/yandexcloud/AUTHORS deleted file mode 100644 index 06ab95beb95..00000000000 --- a/contrib/python/yandexcloud/AUTHORS +++ /dev/null @@ -1,4 +0,0 @@ -The following authors have created the source code of "Yandex.Cloud Python Client Library" published and distributed by YANDEX LLC as the owner: - -Ivan Romanenko <[email protected]> -Islam Alibekov <[email protected]>
\ No newline at end of file diff --git a/contrib/python/yandexcloud/LICENSE b/contrib/python/yandexcloud/LICENSE deleted file mode 100644 index 0cd74fabf04..00000000000 --- a/contrib/python/yandexcloud/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 YANDEX LLC - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/contrib/python/yandexcloud/README.md b/contrib/python/yandexcloud/README.md deleted file mode 100644 index bd8014faa68..00000000000 --- a/contrib/python/yandexcloud/README.md +++ /dev/null @@ -1,177 +0,0 @@ -[![PyPI Version][pypi-image]][pypi-url] -[![Build Status][build-image]][build-url] -[![License][license-image]][license-url] - -<!-- Badges --> - -[pypi-image]: https://img.shields.io/pypi/v/yandexcloud -[pypi-url]: https://pypi.org/project/yandexcloud/ -[build-image]: https://github.com/yandex-cloud/python-sdk/actions/workflows/run-tests.yml/badge.svg -[build-url]: https://github.com/yandex-cloud/python-sdk/actions/workflows/run-tests.yml -[license-image]: https://img.shields.io/github/license/yandex-cloud/python-sdk.svg -[license-url]: https://github.com/yandex-cloud/python-sdk/blob/master/LICENSE - -# Yandex.Cloud SDK (Python) - -Need to automate your infrastructure or use services provided by Yandex.Cloud? We've got you covered. - -Installation: - - pip install yandexcloud - -## Getting started - -There are several options for authorization your requests - OAuth Token, -Metadata Service (if you're executing your code inside VMs or Cloud Functions -running in Yandex.Cloud), Service Account Keys, and externally created IAM tokens. - -### OAuth Token - -```python -sdk = yandexcloud.SDK(token='AQAD-.....') -``` - -### Metadata Service - -Don't forget to assign Service Account for your Instance or Function and grant required roles. - -```python -sdk = yandexcloud.SDK() -``` - -### Service Account Keys - -```python -# you can store and read it from JSON file -sa_key = { - "id": "...", - "service_account_id": "...", - "private_key": "..." -} - -sdk = yandexcloud.SDK(service_account_key=sa_key) -``` - -### IAM tokens - -```python -sdk = yandexcloud.SDK(iam_token="t1.9eu...") -``` - -Check `examples` directory for more examples. - -### Override service endpoint - -#### Supported services - -| Service Name | Alias | -|------------------------------------------------------------------------|--------------------------| -| yandex.cloud.ai.foundation_models | ai-foundation-models | -| yandex.cloud.ai.llm | ai-llm | -| yandex.cloud.ai.stt | ai-stt | -| yandex.cloud.ai.translate | ai-translate | -| yandex.cloud.ai.tts | ai-speechkit | -| yandex.cloud.ai.vision | ai-vision | -| yandex.cloud.apploadbalancer | alb | -| yandex.cloud.billing | billing | -| yandex.cloud.cdn | cdn | -| yandex.cloud.certificatemanager.v1.certificate_content_service | certificate-manager-data | -| yandex.cloud.certificatemanager | certificate-manager | -| yandex.cloud.compute | compute | -| yandex.cloud.containerregistry | container-registry | -| yandex.cloud.dataproc.manager | dataproc-manager | -| yandex.cloud.dataproc | dataproc | -| yandex.cloud.datatransfer | datatransfer | -| yandex.cloud.dns | dns | -| yandex.cloud.endpoint | endpoint | -| yandex.cloud.iam | iam | -| yandex.cloud.iot.devices | iot-devices | -| yandex.cloud.k8s | managed-kubernetes | -| yandex.cloud.kms | kms | -| yandex.cloud.kms.v1.symmetric_crypto_service | kms-crypto | -| yandex.cloud.loadbalancer | load-balancer | -| yandex.cloud.loadtesting | loadtesting | -| yandex.cloud.lockbox.v1.payload_service | lockbox-payload | -| yandex.cloud.lockbox | lockbox | -| yandex.cloud.logging.v1.log_ingestion_service | log-ingestion | -| yandex.cloud.logging.v1.log_reading_service | log-reading | -| yandex.cloud.logging | logging | -| yandex.cloud.marketplace | marketplace | -| yandex.cloud.mdb.clickhouse | managed-clickhouse | -| yandex.cloud.mdb.elasticsearch | managed-elasticsearch | -| yandex.cloud.mdb.greenplum | managed-greenplum | -| yandex.cloud.mdb.kafka | managed-kafka | -| yandex.cloud.mdb.mongodb | managed-mongodb | -| yandex.cloud.mdb.mysql | managed-mysql | -| yandex.cloud.mdb.opensearch | managed-opensearch | -| yandex.cloud.mdb.postgresql | managed-postgresql | -| yandex.cloud.mdb.redis | managed-redis | -| yandex.cloud.mdb.sqlserver | managed-sqlserver | -| yandex.cloud.operation | operation | -| yandex.cloud.organizationmanager | organization-manager | -| yandex.cloud.resourcemanager | resource-manager | -| yandex.cloud.serverless.apigateway | serverless-apigateway | -| yandex.cloud.serverless.apigateway.websocket | apigateway-connections | -| yandex.cloud.serverless.containers | serverless-containers | -| yandex.cloud.serverless.functions | serverless-functions | -| yandex.cloud.serverless.triggers | serverless-triggers | -| yandex.cloud.storage | storage-api | -| yandex.cloud.vpc | vpc | -| yandex.cloud.ydb | ydb | - - -#### Override in client -```python -from yandex.cloud.vpc.v1.network_service_pb2_grpc import NetworkServiceStub -from yandexcloud import SDK - -sdk = SDK(iam_token="t1.9eu...") -new_network_client_endpoint = "example.new.vpc.very.new.yandex:50051" -insecure = False # by default is False, but if server does not support verification can be set to True -network_client = sdk.client(NetworkServiceStub, endpoint=new_network_client_endpoint, insecure=False) -``` - -#### Override in sdk config -To override endpoints provide dict in format {alias : new-endpoint} -```python -from yandex.cloud.vpc.v1.network_service_pb2_grpc import NetworkServiceStub -from yandexcloud import SDK -new_network_client_endpoint = "example.new.vpc.very.new.yandex:50051" -sdk = SDK(iam_token="t1.9eu...", endpoints={"vpc": new_network_client_endpoint}) -insecure = False # by default is False, but if server does not support verification can be set to True -network_client = sdk.client(NetworkServiceStub, insecure=False) -``` - -Notice: if both overrides are used for same endpoint, override by client has priority - -#### Switch SDK region -```python -from yandexcloud import SDK, set_up_yc_api_endpoint -kz_region_endpoint = "api.yandexcloud.kz" -# this will make SDK list endpoints from KZ yc installation -sdk = SDK(iam_token="t1.9eu...", endpoint="api.yandexcloud.kz") -# or you can use global function -set_up_yc_api_endpoint(kz_region_endpoint) -``` - -## Contributing -### Dependencies -Use `make deps` command to install library, its production and development dependencies. - -### Formatting -Use `make format` to autoformat code with black tool. - -### Tests -- `make test` to run tests for current python version -- `make lint` to run only linters for current python version -- `make tox-current` to run all checks (tests + code style checks + linters + format check) for current python version -- `make tox` to run all checks for all supported (installed in your system) python versions -- `make test-all-versions` to run all checks for all supported python versions in docker container - - -### Maintaining -If pull request consists of several meaningful commits, that should be preserved, -then use "Rebase and merge" option. Otherwise use "Squash and merge". - -New release (changelog, tag and pypi upload) will be automatically created -on each push to master via Github Actions workflow. diff --git a/contrib/python/yandexcloud/ya.make b/contrib/python/yandexcloud/ya.make deleted file mode 100644 index 3815d07fae3..00000000000 --- a/contrib/python/yandexcloud/ya.make +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(0.293.0) - -LICENSE(MIT) - -PEERDIR( - contrib/libs/googleapis-common-protos - contrib/libs/yandex-cloud-api-protos - contrib/python/PyJWT - contrib/python/cryptography - contrib/python/grpcio - contrib/python/protobuf - contrib/python/requests - contrib/python/six -) - -NO_LINT() - -PY_SRCS( - TOP_LEVEL - yandexcloud/__init__.py - yandexcloud/_auth_fabric.py - yandexcloud/_auth_plugin.py - yandexcloud/_backoff.py - yandexcloud/_channels.py - yandexcloud/_helpers.py - yandexcloud/_operation_waiter.py - yandexcloud/_retry_interceptor.py - yandexcloud/_sdk.py - yandexcloud/_wrappers/__init__.py - yandexcloud/_wrappers/dataproc/__init__.py - yandexcloud/auth.py - yandexcloud/operations.py -) - -RESOURCE_FILES( - PREFIX contrib/python/yandexcloud/ - .dist-info/METADATA - .dist-info/top_level.txt - yandexcloud/py.typed -) - -END() diff --git a/contrib/python/yandexcloud/yandexcloud/__init__.py b/contrib/python/yandexcloud/yandexcloud/__init__.py deleted file mode 100644 index b036a685492..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Main package for Yandex.Cloud SDK.""" - -# flake8: noqa -from yandexcloud._auth_fabric import set_up_yc_api_endpoint -from yandexcloud._backoff import ( - backoff_exponential_with_jitter, - backoff_linear_with_jitter, - default_backoff, -) -from yandexcloud._retry_interceptor import RetryInterceptor -from yandexcloud._sdk import SDK - -__version__ = "0.0.2" diff --git a/contrib/python/yandexcloud/yandexcloud/_auth_fabric.py b/contrib/python/yandexcloud/yandexcloud/_auth_fabric.py deleted file mode 100644 index 135dab40204..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_auth_fabric.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import time -from datetime import datetime -from typing import Dict, Optional, Union - -# noinspection PyUnresolvedReferences -# jwt package depends on cryptography -import cryptography # noqa: F401; pylint: disable=unused-import -import jwt -import requests -import six - -from yandex.cloud.iam.v1.iam_token_service_pb2 import CreateIamTokenRequest - -_MDS_ADDR = "169.254.169.254" -_MDS_URL = "http://{}/computeMetadata/v1/instance/service-accounts/default/token" -_MDS_HEADERS = {"Metadata-Flavor": "Google"} -_MDS_TIMEOUT = (1.0, 1.0) # 1sec connect, 1sec read - -YC_API_ENDPOINT = "api.cloud.yandex.net" - - -def set_up_yc_api_endpoint(endpoint: str) -> str: - # pylint: disable-next=global-statement - global YC_API_ENDPOINT - YC_API_ENDPOINT = endpoint - return YC_API_ENDPOINT - - -def __validate_service_account_key(sa_key: Optional[dict]) -> bool: - if not isinstance(sa_key, dict): - raise RuntimeError(f"Invalid Service Account Key: expecting dictionary, actually got {type(sa_key)}") - - obj_id = sa_key.get("id") - sa_id = sa_key.get("service_account_id") - private_key = sa_key.get("private_key") - - if not obj_id: - raise RuntimeError("Invalid Service Account Key: missing key object id.") - - if not sa_id: - raise RuntimeError("Invalid Service Account Key: missing service account id.") - - if not private_key: - raise RuntimeError("Invalid Service Account Key: missing private key.") - - private_key_prefix = "-----BEGIN PRIVATE KEY-----" - if not isinstance(private_key, six.string_types) or private_key_prefix not in private_key: - error_message = ( - "Invalid Service Account Key: private key is in incorrect format." - f"Should start with {private_key_prefix}.\n" - "To obtain one you can use YC CLI: yc iam key create --output sa.json --service-account-id <id>" - ) - raise RuntimeError(error_message) - return True - - -class MetadataAuth: - def __init__(self, metadata_addr: str): - self.__metadata_addr = metadata_addr - - def url(self) -> str: - return _MDS_URL.format(self.__metadata_addr) - - def get_token(self) -> str: - r = requests.get(self.url(), headers=_MDS_HEADERS, timeout=_MDS_TIMEOUT) - r.raise_for_status() - response = r.json() - return response["access_token"] - - -class TokenAuth: - def __init__(self, token: str): - self.__oauth_token = token - - def get_token_request(self) -> "CreateIamTokenRequest": - return CreateIamTokenRequest(yandex_passport_oauth_token=self.__oauth_token) - - -class ServiceAccountAuth: - __SECONDS_IN_HOUR = 60.0 * 60.0 - - def __init__(self, sa_key: Dict[str, str], endpoint: Optional[str] = None): - self.__sa_key = sa_key - self._endpoint = endpoint if endpoint is not None else YC_API_ENDPOINT - - def get_token_request(self) -> "CreateIamTokenRequest": - return CreateIamTokenRequest(jwt=self.__prepare_request(self._endpoint)) - - def __prepare_request(self, endpoint: str) -> str: - now = time.time() - now_utc = datetime.utcfromtimestamp(now) - exp_utc = datetime.utcfromtimestamp(now + self.__SECONDS_IN_HOUR) - url = f"https://iam.{endpoint}/iam/v1/tokens" - payload = { - "iss": self.__sa_key["service_account_id"], - "aud": url, - "iat": now_utc, - "exp": exp_utc, - } - - headers = { - "typ": "JWT", - "alg": "PS256", - "kid": self.__sa_key["id"], - } - - return jwt.encode(payload, self.__sa_key["private_key"], algorithm="PS256", headers=headers) - - -class IamTokenAuth: - def __init__(self, iam_token: str): - self.__iam_token = iam_token - - def get_token(self) -> str: - return self.__iam_token - - -def get_auth_token_requester( - token: Optional[str] = None, - service_account_key: Optional[dict] = None, - iam_token: Optional[str] = None, - metadata_addr: Optional[str] = None, - endpoint: Optional[str] = None, -) -> Union["MetadataAuth", "TokenAuth", "IamTokenAuth", "ServiceAccountAuth"]: - if endpoint is None: - endpoint = YC_API_ENDPOINT - auth_methods = [("token", token), ("service_account_key", service_account_key), ("iam_token", iam_token)] - auth_methods = [(auth_type, value) for auth_type, value in auth_methods if value is not None] - - if len(auth_methods) == 0: - metadata_addr = metadata_addr if metadata_addr is not None else os.environ.get("YC_METADATA_ADDR", _MDS_ADDR) - return MetadataAuth(metadata_addr) - - if len(auth_methods) > 1: - raise RuntimeError(f"Conflicting API credentials properties are set: {[auth[0] for auth in auth_methods]}.") - - if token is not None: - return TokenAuth(token=token) - if iam_token is not None: - return IamTokenAuth(iam_token) - if service_account_key is not None and __validate_service_account_key(service_account_key): - return ServiceAccountAuth(service_account_key, endpoint) - - raise RuntimeError("Unknown auth method") diff --git a/contrib/python/yandexcloud/yandexcloud/_auth_plugin.py b/contrib/python/yandexcloud/yandexcloud/_auth_plugin.py deleted file mode 100644 index 46e724b31f3..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_auth_plugin.py +++ /dev/null @@ -1,96 +0,0 @@ -from datetime import datetime -from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union - -import grpc -from six.moves.urllib.parse import urlparse - -from yandex.cloud.iam.v1.iam_token_service_pb2_grpc import IamTokenServiceStub - -if TYPE_CHECKING: - from yandex.cloud.iam.v1.iam_token_service_pb2 import CreateIamTokenResponse - from yandexcloud._auth_fabric import ( - IamTokenAuth, - MetadataAuth, - ServiceAccountAuth, - TokenAuth, - ) - - -TIMEOUT_SECONDS = 20 - - -class Credentials(grpc.AuthMetadataPlugin): - def __init__( - self, - token_requester: Union["MetadataAuth", "TokenAuth", "IamTokenAuth", "ServiceAccountAuth"], - lazy_channel: Callable[[], "grpc.Channel"], - ): - # pylint: disable=super-init-not-called - self.__token_requester = token_requester - self._lazy_channel = lazy_channel - self._channel: Optional[grpc.Channel] = None - self._cached_iam_token: str = "" - self._iam_token_timestamp: Optional[datetime] = None - - def __call__(self, context: "grpc.AuthMetadataContext", callback: "grpc.AuthMetadataPluginCallback") -> None: - try: - return self._call(context, callback) - except Exception as exception: # pylint: disable=broad-except - callback(tuple(), exception) - return None - - def _call(self, context: "grpc.AuthMetadataContext", callback: "grpc.AuthMetadataPluginCallback") -> None: - u = urlparse(context.service_url) - if u.path in ( - "/yandex.cloud.iam.v1.IamTokenService", - "/yandex.cloud.endpoint.ApiEndpointService", - ): - callback(tuple(), None) - return - - if self._channel is None: - self._channel = self._lazy_channel() - - if not self._fresh(): - get_token = getattr(self.__token_requester, "get_token", None) - if callable(get_token): - self._cached_iam_token = get_token() - self._iam_token_timestamp = datetime.now() - callback(self._metadata(), None) - return - - get_token_request = getattr(self.__token_requester, "get_token_request", None) - if callable(get_token_request): - token_future = IamTokenServiceStub(self._channel).Create.future(get_token_request()) - token_future.add_done_callback(self.create_done_callback(callback)) - return - - callback(self._metadata(), None) - - def create_done_callback(self, callback: "grpc.AuthMetadataPluginCallback") -> Callable[["grpc.Future"], None]: - def done_callback(future: "grpc.Future") -> None: - try: - resp = future.result() - except Exception as exception: # pylint: disable=broad-except - callback(tuple(), exception) - else: - self._save_token(resp) - callback(self._metadata(), None) - - return done_callback - - def _metadata(self) -> Tuple[Tuple[str, str]]: - metadata = (("authorization", f"Bearer {self._cached_iam_token}"),) - return metadata - - def _save_token(self, resp: "CreateIamTokenResponse") -> None: - self._cached_iam_token = resp.iam_token - self._iam_token_timestamp = datetime.now() - - def _fresh(self) -> bool: - if self._cached_iam_token == "": - return False - if self._iam_token_timestamp is None: - return False - diff = datetime.now() - self._iam_token_timestamp - return diff.total_seconds() < TIMEOUT_SECONDS diff --git a/contrib/python/yandexcloud/yandexcloud/_backoff.py b/contrib/python/yandexcloud/yandexcloud/_backoff.py deleted file mode 100644 index 0ed4bdd9e4f..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_backoff.py +++ /dev/null @@ -1,45 +0,0 @@ -import random -from typing import Callable - - -def backoff_linear_with_jitter(wait_time: float, jitter: float) -> Callable[[int], float]: - def func(attempt: int) -> float: # pylint: disable=unused-argument - multiplier = jitter * (random.random() * 2 - 1) - return wait_time * (1 + multiplier) - - return func - - -def backoff_exponential_with_jitter(base: float, cap: float) -> Callable[[int], float]: - def func(attempt: int) -> float: # pylint: disable=unused-argument - try: - res = (2**attempt) * base * random.random() - except OverflowError: - return cap - - if res > cap: - return cap - - return res - - return func - - -def backoff_exponential_jittered_min_interval(base: float = 0.05, cap: float = 60) -> Callable[[int], float]: - def func(attempt: int) -> float: # pylint: disable=unused-argument - try: - base_interval = (2**attempt) * base - res = base_interval / 2 + base_interval * random.random() - except OverflowError: - return cap - - if res > cap: - return cap - - return res - - return func - - -def default_backoff() -> Callable[[int], float]: - return backoff_exponential_with_jitter(0.05, 60) diff --git a/contrib/python/yandexcloud/yandexcloud/_channels.py b/contrib/python/yandexcloud/yandexcloud/_channels.py deleted file mode 100644 index 73aa3f9004a..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_channels.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging -from importlib.metadata import PackageNotFoundError, version -from typing import Dict, Optional - -import grpc - -from yandex.cloud.endpoint.api_endpoint_service_pb2 import ListApiEndpointsRequest -from yandex.cloud.endpoint.api_endpoint_service_pb2_grpc import ApiEndpointServiceStub -from yandexcloud import _auth_plugin -from yandexcloud._auth_fabric import YC_API_ENDPOINT, get_auth_token_requester - -try: - VERSION = version("yandexcloud") -except PackageNotFoundError: - VERSION = "0.0.0" - -SDK_USER_AGENT = f"yandex-cloud-python-sdk/{VERSION}" -logger = logging.getLogger(__name__) - - -class Channels: - def __init__( - self, - client_user_agent: Optional[str] = None, - endpoints: Optional[Dict[str, str]] = None, - token: Optional[str] = None, - iam_token: Optional[str] = None, - endpoint: Optional[str] = None, - service_account_key: Optional[Dict[str, str]] = None, - root_certificates: Optional[bytes] = None, - private_key: Optional[bytes] = None, - certificate_chain: Optional[bytes] = None, - **_: str, - ) -> None: - self._channel_creds = grpc.ssl_channel_credentials( - root_certificates=root_certificates, - private_key=private_key, - certificate_chain=certificate_chain, - ) - self._endpoint = endpoint if endpoint is not None else YC_API_ENDPOINT - self._token_requester = get_auth_token_requester( - token=token, - service_account_key=service_account_key, - iam_token=iam_token, - endpoint=self._endpoint, - ) - - self._client_user_agent = client_user_agent - self._config_endpoints = endpoints if endpoints is not None else {} - self._endpoints: Optional[Dict[str, str]] = None - self.channel_options = tuple( - ("grpc.primary_user_agent", user_agent) - for user_agent in [self._client_user_agent, SDK_USER_AGENT] - if user_agent is not None - ) - - def channel(self, service: str, endpoint: Optional[str] = None, insecure: bool = False) -> grpc.Channel: - if endpoint: - logger.info("Using provided service %s endpoint %s", service, endpoint) - if insecure: - logger.info("Insecure option is ON, no IAM endpoint used for verification") - return grpc.insecure_channel(endpoint, options=self.channel_options) - logger.info("Insecure option is OFF,IAM endpoint %s used for verification") - creds: grpc.ChannelCredentials = self._get_creds(self.endpoints["iam"]) - return grpc.secure_channel(endpoint, creds, options=self.channel_options) - if service not in self._config_endpoints and insecure: - logger.warning( - "Unable to use insecure option for default {%s} service endpoint.\n" - "Option is ignored. To enable it override endpoint.", - service, - ) - elif insecure: - logger.info("Insecure option is ON, no IAM endpoint used for verification") - return grpc.insecure_channel(self.endpoints[service], options=self.channel_options) - - logger.info( - "Using endpoints from configuration, IAM %s, %s %s", - self.endpoints["iam"], - service, - self.endpoints[service], - ) - - creds = self._get_creds(self.endpoints["iam"]) - if service not in self.endpoints: - raise RuntimeError(f"Unknown service: {service}") - return grpc.secure_channel(self.endpoints[service], creds, options=self.channel_options) - - @property - def endpoints(self) -> Dict[str, str]: - if self._endpoints is None: - self._endpoints = self._get_endpoints() - for id_, address in self._config_endpoints.items(): - logger.debug("Override service %s, endpoint %s", id_, address) - if id_ == "iam": - logger.warning( - "Be aware `iam` service endpoint is overridden. " - "That can produce unexpected results in SDK calls." - ) - self._endpoints[id_] = address - return self._endpoints - - def _get_endpoints(self) -> Dict[str, str]: - unauthenticated_channel = grpc.secure_channel(self._endpoint, self._channel_creds, options=self.channel_options) - endpoint_service = ApiEndpointServiceStub(unauthenticated_channel) - resp = endpoint_service.List(ListApiEndpointsRequest()) - return {endpoint.id: endpoint.address for endpoint in resp.endpoints} - - def _get_creds(self, iam_endpoint: str) -> grpc.ChannelCredentials: - plugin = _auth_plugin.Credentials( - self._token_requester, lambda: grpc.secure_channel(iam_endpoint, creds, options=self.channel_options) - ) - call_creds = grpc.metadata_call_credentials(plugin) - creds = grpc.composite_channel_credentials(self._channel_creds, call_creds) - return creds diff --git a/contrib/python/yandexcloud/yandexcloud/_helpers.py b/contrib/python/yandexcloud/yandexcloud/_helpers.py deleted file mode 100644 index 0df9b6c03cd..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_helpers.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -from typing import TYPE_CHECKING, Optional - -from yandex.cloud.iam.v1.service_account_service_pb2 import ListServiceAccountsRequest -from yandex.cloud.iam.v1.service_account_service_pb2_grpc import ( - ServiceAccountServiceStub, -) -from yandex.cloud.vpc.v1.network_service_pb2 import ListNetworksRequest -from yandex.cloud.vpc.v1.network_service_pb2_grpc import NetworkServiceStub -from yandex.cloud.vpc.v1.subnet_service_pb2 import ListSubnetsRequest -from yandex.cloud.vpc.v1.subnet_service_pb2_grpc import SubnetServiceStub - -if TYPE_CHECKING: - from yandexcloud._sdk import SDK - - -class Helpers: - def __init__(self, sdk: "SDK"): - self.sdk = sdk - - def find_service_account_id(self, folder_id: str) -> str: - """ - Get service account id in case the folder has the only one service account - - :param folder_id: ID of the folder - :return ID of the service account - """ - service = self.sdk.client(ServiceAccountServiceStub) - service_accounts = service.List(ListServiceAccountsRequest(folder_id=folder_id)).service_accounts - if len(service_accounts) == 1: - return service_accounts[0].id - if len(service_accounts) == 0: - raise RuntimeError(f"There are no service accounts in folder {folder_id}, please create it.") - raise RuntimeError(f"There are more than one service account in folder {folder_id}, please specify it") - - def find_network_id(self, folder_id: str) -> str: - """ - Get ID of the first network in folder - - :param folder_id: ID of the folder - :return ID of the network - """ - networks = self.sdk.client(NetworkServiceStub).List(ListNetworksRequest(folder_id=folder_id)).networks - if not networks: - raise RuntimeError(f"No networks in folder: {folder_id}") - if len(networks) > 1: - raise RuntimeError("There are more than one network in folder {folder_id}, please specify it") - return networks[0].id - - def find_subnet_id(self, folder_id: str, zone_id: str, network_id: Optional[str] = None) -> str: - """ - Get ID of the subnetwork of specified network in specified availability zone - - :param folder_id: ID of the folder - :param zone_id: ID of the availability zone - :param network_id: ID of the network - :return ID of the subnetwork - """ - subnet_service = self.sdk.client(SubnetServiceStub) - subnets = subnet_service.List(ListSubnetsRequest(folder_id=folder_id)).subnets - if network_id: - applicable = [s for s in subnets if s.zone_id == zone_id and s.network_id == network_id] - else: - applicable = [s for s in subnets if s.zone_id == zone_id] - if len(applicable) == 1: - return applicable[0].id - if len(applicable) == 0: - raise RuntimeError(f"There are no subnets in {zone_id} zone, please create it.") - raise RuntimeError(f"There are more than one subnet in {zone_id} zone, please specify it") diff --git a/contrib/python/yandexcloud/yandexcloud/_operation_waiter.py b/contrib/python/yandexcloud/yandexcloud/_operation_waiter.py deleted file mode 100644 index 6c321e235cc..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_operation_waiter.py +++ /dev/null @@ -1,128 +0,0 @@ -import logging -import time -from datetime import datetime -from typing import TYPE_CHECKING, Optional, Type, Union - -import grpc -from google.protobuf.empty_pb2 import Empty - -from yandex.cloud.operation.operation_service_pb2 import GetOperationRequest -from yandex.cloud.operation.operation_service_pb2_grpc import OperationServiceStub -from yandexcloud._backoff import backoff_exponential_jittered_min_interval -from yandexcloud._retry_interceptor import RetryInterceptor -from yandexcloud.operations import OperationError, OperationResult - -if TYPE_CHECKING: - import google.protobuf.message - - from yandex.cloud.operation.operation_pb2 import Operation - from yandexcloud._sdk import SDK - - -def operation_waiter(sdk: "SDK", operation_id: str, timeout: Optional[float]) -> "OperationWaiter": - retriable_codes = ( - grpc.StatusCode.UNAVAILABLE, - grpc.StatusCode.RESOURCE_EXHAUSTED, - grpc.StatusCode.INTERNAL, - ) - # withstand server downtime for ~3.4 minutes with an exponential backoff - retry_interceptor = RetryInterceptor( - max_retry_count=13, - per_call_timeout=30, - back_off_func=backoff_exponential_jittered_min_interval(), - retriable_codes=retriable_codes, - ) - operation_service = sdk.client( - OperationServiceStub, - interceptor=retry_interceptor, - ) - return OperationWaiter(operation_id, operation_service, timeout) - - -def wait_for_operation(sdk: "SDK", operation_id: str, timeout: Optional[float]) -> Optional["Operation"]: - waiter = operation_waiter(sdk, operation_id, timeout) - for _ in waiter: - time.sleep(1) - return waiter.operation - - -def get_operation_result( - sdk: "SDK", - operation: "Operation", - response_type: Optional[Type["google.protobuf.message.Message"]] = None, - meta_type: Optional[Type["google.protobuf.message.Message"]] = None, - timeout: Optional[float] = None, - logger: Optional[logging.Logger] = None, -) -> Union["OperationResult", "OperationError"]: - if not logger: - logger = logging.getLogger() - logger.addHandler(logging.NullHandler()) - operation_result = OperationResult(operation) - created_at = datetime.fromtimestamp(operation.created_at.seconds) - message = ( - "Running Yandex.Cloud operation. ID: {id}. " - "Description: {description}. Created at: {created_at}. " - "Created by: {created_by}." - ) - message = message.format( - id=operation.id, - description=operation.description, - created_at=created_at, - created_by=operation.created_by, - ) - if meta_type and meta_type is not Empty: - unpacked_meta = meta_type() - operation.metadata.Unpack(unpacked_meta) - operation_result.meta = unpacked_meta - message += f" Meta: {unpacked_meta}." - logger.info(message) - result = wait_for_operation(sdk, operation.id, timeout=timeout) - if result is None: - return OperationError(message="Unexpected operation result", operation_result=OperationResult(operation)) - if result.error and result.error.code: - error_message = ( - "Error Yandex.Cloud operation. ID: {id}. Error code: {code}. Details: {details}. Message: {message}." - ) - error_message = error_message.format( - id=result.id, - code=result.error.code, - details=result.error.details, - message=result.error.message, - ) - logger.error(error_message) - raise OperationError(message=error_message, operation_result=OperationResult(operation)) - - log_message = f"Done Yandex.Cloud operation. ID: {operation.id}." - if response_type and response_type is not Empty: - unpacked_response = response_type() - result.response.Unpack(unpacked_response) - operation_result.response = unpacked_response - log_message += f" Response: {unpacked_response}." - logger.info(log_message) - return operation_result - - -class OperationWaiter: - def __init__(self, operation_id: str, operation_service: "OperationServiceStub", timeout: Optional[float] = None): - self.__operation: Optional["Operation"] = None - self.__operation_id = operation_id - self.__operation_service = operation_service - self.__deadline = time.time() + timeout if timeout else None - - @property - def operation(self) -> Optional["Operation"]: - return self.__operation - - @property - def done(self) -> bool: - self.__operation = self.__operation_service.Get(GetOperationRequest(operation_id=self.__operation_id)) - return self.__operation is not None and self.__operation.done - - def __iter__(self) -> "OperationWaiter": - return self - - def __next__(self) -> None: - if self.done or self.__deadline is not None and time.time() >= self.__deadline: - raise StopIteration() - - next = __next__ # for Python 2 diff --git a/contrib/python/yandexcloud/yandexcloud/_retry_interceptor.py b/contrib/python/yandexcloud/yandexcloud/_retry_interceptor.py deleted file mode 100644 index d3261f01136..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_retry_interceptor.py +++ /dev/null @@ -1,208 +0,0 @@ -import collections -import time -import uuid -from typing import Callable, Iterable, Optional - -import grpc - - -class _ClientCallDetails( - collections.namedtuple( - "_ClientCallDetails", ("method", "timeout", "metadata", "credentials", "wait_for_ready", "compression") - ), - grpc.ClientCallDetails, -): - pass - - -class _RetryCall(Exception): - pass - - -class RetryInterceptor(grpc.UnaryUnaryClientInterceptor): - """RetryInterceptor implements grpc retries. - It supports retries quantity, list of retriable codes, backoff function, - per retry call timeout, and writing retry attempt to call metadata. - Important nodes: - 1. If default parameters are used, no retries will be executed. - 2. It will always add idempotency token to grpc call metadata, if one is not already present. - 3. Negative max_retry_count parameter will result in INFINITE retries. - 4. DEADLINE_EXCEEDED and CANCELLED are not retriable codes. - 5. Default retriable codes are UNAVAILABLE and RESOURCE_EXHAUSTED. - 6. Backoff function is called with retry attempt counter and should return sleep time in seconds (float). - """ - - _DEFAULT_RETRIABLE_CODES = ( - grpc.StatusCode.UNAVAILABLE, - grpc.StatusCode.RESOURCE_EXHAUSTED, - ) - _NON_RETRIABLE_CODES = [grpc.StatusCode.CANCELLED, grpc.StatusCode.DEADLINE_EXCEEDED] - _IDEMPOTENCY_TOKEN_METADATA_KEY = "idempotency-key" - _ATTEMPT_METADATA_KEY = "x-retry-attempt" - - def __init__( - self, - max_retry_count: int = 0, - retriable_codes: Iterable["grpc.StatusCode"] = _DEFAULT_RETRIABLE_CODES, - add_retry_count_to_header: bool = False, - back_off_func: Optional[Callable[[int], float]] = None, - per_call_timeout: Optional[float] = None, - ): - # pylint: disable=super-init-not-called - self.__max_retry_count = max_retry_count - self.__retriable_codes = retriable_codes - self.__add_retry_count_to_header = add_retry_count_to_header - self.__back_off_func = back_off_func - self.__per_call_timeout = per_call_timeout - - def intercept_unary_unary( - self, - continuation: Callable[["grpc.ClientCallDetails", "grpc.TRequest"], "grpc.CallFuture[grpc.TResponse]"], - client_call_details: "grpc.ClientCallDetails", - request: "grpc.TRequest", - ) -> "grpc.CallFuture[grpc.TResponse]": - client_call_details = self.__add_idempotency_token(client_call_details) - - attempt = 0 - deadline = self.__deadline(client_call_details.timeout) - - while True: - try: - return self.__grpc_call(attempt, deadline, continuation, client_call_details, request) - except _RetryCall: - attempt += 1 - - def __wait_backoff(self, attempt: int, deadline: Optional[float]) -> None: - if self.__back_off_func is None: - return - - backoff_timeout = self.__back_off_func(attempt) - - if deadline is not None: - deadline_timeout = deadline - time.time() - - if backoff_timeout > deadline_timeout: # pylint: disable=consider-using-min-builtin - backoff_timeout = deadline_timeout - - if backoff_timeout > 0.0: - time.sleep(backoff_timeout) - - @staticmethod - def __deadline(timeout: Optional[float]) -> Optional[float]: - return time.time() + timeout if timeout is not None else None - - def __is_retriable(self, error: "grpc.StatusCode") -> bool: - if error in self._NON_RETRIABLE_CODES: - return False - - if error in self.__retriable_codes: - return True - - return False - - @staticmethod - def __min_deadline(d1: Optional[float], d2: Optional[float]) -> Optional[float]: - if d2 is None and d1 is None: - return None - if d1 is None: - return d2 - - if d2 is None: - return d1 - - return min(d1, d2) - - def __grpc_call( - self, - attempt: int, - deadline: Optional[float], - continuation: Callable[["grpc.ClientCallDetails", "grpc.TRequest"], "grpc.CallFuture[grpc.TResponse]"], - client_call_details: "grpc.ClientCallDetails", - request: "grpc.TRequest", - ) -> "grpc.CallFuture[grpc.TResponse]": - if attempt > 0: - if self.__add_retry_count_to_header: - client_call_details = self.__append_retry_attempt_header(client_call_details, attempt) - - call_deadline = self.__deadline(self.__per_call_timeout) - call_deadline = self.__min_deadline(deadline, call_deadline) - - if call_deadline is not None: - client_call_details = self.__adjust_timeout(client_call_details, call_deadline) - - def retry() -> None: - self.__wait_backoff(attempt, deadline) - raise _RetryCall() - - try: - result = continuation(client_call_details, request) - if isinstance(result, grpc.RpcError): # type: ignore - raise result - return result - except grpc.RpcError as error: - # no retries left - if 0 <= self.__max_retry_count <= attempt: - raise - - err_code = error.code() # pylint: disable=no-member - if err_code == grpc.StatusCode.DEADLINE_EXCEEDED: - # if there is no per_call_timeout, or it is original deadline -> abort, otherwise, retry call. - if self.__per_call_timeout is None or deadline is not None and deadline < time.time(): - raise - - retry() - - if not self.__is_retriable(err_code): - raise - - retry() - raise RuntimeError("Unexpected behavior") - - @staticmethod - def __adjust_timeout(client_call_details: "grpc.ClientCallDetails", deadline: float) -> "grpc.ClientCallDetails": - timeout = max(deadline - time.time(), 0.0) - return _ClientCallDetails( - client_call_details.method, - timeout, - client_call_details.metadata, - client_call_details.credentials, - getattr(client_call_details, "wait_for_ready", None), - getattr(client_call_details, "compression", None), - ) - - def __add_idempotency_token(self, client_call_details: "grpc.ClientCallDetails") -> "grpc.ClientCallDetails": - return self.__append_metadata(client_call_details, self._IDEMPOTENCY_TOKEN_METADATA_KEY, str(uuid.uuid4())) - - def __append_retry_attempt_header( - self, client_call_details: "grpc.ClientCallDetails", attempt: int - ) -> "grpc.ClientCallDetails": - return self.__append_metadata(client_call_details, self._ATTEMPT_METADATA_KEY, str(attempt), force=True) - - @staticmethod - def __append_metadata( - client_call_details: "grpc.ClientCallDetails", header: str, value: str, force: bool = False - ) -> "grpc.ClientCallDetails": - metadata = [] - if client_call_details.metadata is not None: - metadata = list(client_call_details.metadata) - - if not force: - # Do not add value, if there is already one. - for item in metadata: - if item[0] == header: - return client_call_details - - metadata.append( - ( - header, - value, - ) - ) - return _ClientCallDetails( - client_call_details.method, - client_call_details.timeout, - metadata, - client_call_details.credentials, - getattr(client_call_details, "wait_for_ready", None), - getattr(client_call_details, "compression", None), - ) diff --git a/contrib/python/yandexcloud/yandexcloud/_sdk.py b/contrib/python/yandexcloud/yandexcloud/_sdk.py deleted file mode 100644 index f13c27697f1..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_sdk.py +++ /dev/null @@ -1,193 +0,0 @@ -import inspect -from typing import TYPE_CHECKING, Any, Dict, Optional, Type, Union - -import grpc - -from yandexcloud import _channels, _helpers, _operation_waiter -from yandexcloud._backoff import backoff_exponential_with_jitter -from yandexcloud._retry_interceptor import RetryInterceptor -from yandexcloud._wrappers import Wrappers - -if TYPE_CHECKING: - import logging - - import google.protobuf.message - - from yandex.cloud.operation.operation_pb2 import Operation - from yandexcloud._operation_waiter import OperationWaiter - from yandexcloud.operations import OperationError, OperationResult - - -class SDK: - def __init__( - self, - interceptor: Union[ - grpc.UnaryUnaryClientInterceptor, - grpc.UnaryStreamClientInterceptor, - grpc.StreamUnaryClientInterceptor, - grpc.StreamStreamClientInterceptor, - None, - ] = None, - user_agent: Optional[str] = None, - endpoints: Optional[Dict[str, str]] = None, - token: Optional[str] = None, - iam_token: Optional[str] = None, - endpoint: Optional[str] = None, - service_account_key: Optional[Dict[str, str]] = None, - root_certificates: Optional[bytes] = None, - private_key: Optional[bytes] = None, - certificate_chain: Optional[bytes] = None, - **kwargs: str, - ): - """ - API entry-point object. - - :param interceptor: GRPC interceptor to be used instead of default RetryInterceptor - :param user_agent: String to prepend User-Agent metadata header for all GRPC requests made via SDK object - :param endpoints: Dict with services endpoints overrides. Example: {'vpc': 'new.vpc.endpoint:443'} - - """ - self._channels = _channels.Channels( - user_agent, - endpoints, - token, - iam_token, - endpoint, - service_account_key, - root_certificates, - private_key, - certificate_chain, - **kwargs, - ) - if interceptor is None: - interceptor = RetryInterceptor( - max_retry_count=5, - per_call_timeout=30, - back_off_func=backoff_exponential_with_jitter(1, 30), - ) - self._default_interceptor = interceptor - self.helpers = _helpers.Helpers(self) - self.wrappers = Wrappers(self) - - def client( - self, - stub_ctor: Type, - interceptor: Union[ - grpc.UnaryUnaryClientInterceptor, - grpc.UnaryStreamClientInterceptor, - grpc.StreamUnaryClientInterceptor, - grpc.StreamStreamClientInterceptor, - None, - ] = None, - endpoint: Optional[str] = None, - insecure: bool = False, - ) -> Any: - service = _service_for_ctor(stub_ctor) - channel = self._channels.channel(service, endpoint, insecure) - if interceptor is not None: - channel = grpc.intercept_channel(channel, interceptor) - elif self._default_interceptor is not None: - channel = grpc.intercept_channel(channel, self._default_interceptor) - return stub_ctor(channel) - - def waiter(self, operation_id: str, timeout: Optional[float] = None) -> "OperationWaiter": - return _operation_waiter.operation_waiter(self, operation_id, timeout) - - def wait_operation_and_get_result( - self, - operation: "Operation", - response_type: Optional[Type["google.protobuf.message.Message"]] = None, - meta_type: Optional[Type["google.protobuf.message.Message"]] = None, - timeout: Optional[float] = None, - logger: Optional["logging.Logger"] = None, - ) -> Union["OperationResult", "OperationError"]: - return _operation_waiter.get_operation_result(self, operation, response_type, meta_type, timeout, logger) - - def create_operation_and_get_result( - self, - request: Type["google.protobuf.message.Message"], - service: Any, - method_name: str, - response_type: Optional[Type["google.protobuf.message.Message"]] = None, - meta_type: Optional[Type["google.protobuf.message.Message"]] = None, - timeout: Optional[float] = None, - logger: Optional["logging.Logger"] = None, - ) -> Union["OperationResult", "OperationError"]: - operation = getattr(self.client(service), method_name)(request) - return self.wait_operation_and_get_result( - operation, - response_type=response_type, - meta_type=meta_type, - timeout=timeout, - logger=logger, - ) - - -def _service_for_ctor(stub_ctor: Any) -> str: - m = inspect.getmodule(stub_ctor) - if m is not None: - name = m.__name__ - if not name.startswith("yandex.cloud"): - raise RuntimeError(f"Not a yandex.cloud service {stub_ctor}") - - for k, v in _supported_modules: - if name.startswith(k): - return v - - raise RuntimeError(f"Unknown service {stub_ctor}") - - -_supported_modules = [ - ("yandex.cloud.ai.foundation_models", "ai-foundation-models"), - ("yandex.cloud.ai.llm", "ai-llm"), - ("yandex.cloud.ai.stt", "ai-stt"), - ("yandex.cloud.ai.translate", "ai-translate"), - ("yandex.cloud.ai.tts", "ai-speechkit"), - ("yandex.cloud.ai.vision", "ai-vision"), - ("yandex.cloud.apploadbalancer", "alb"), - ("yandex.cloud.billing", "billing"), - ("yandex.cloud.cdn", "cdn"), - ("yandex.cloud.certificatemanager.v1.certificate_content_service", "certificate-manager-data"), - ("yandex.cloud.certificatemanager", "certificate-manager"), - ("yandex.cloud.compute", "compute"), - ("yandex.cloud.containerregistry", "container-registry"), - ("yandex.cloud.dataproc.manager", "dataproc-manager"), - ("yandex.cloud.dataproc", "dataproc"), - ("yandex.cloud.datatransfer", "datatransfer"), - ("yandex.cloud.dns", "dns"), - ("yandex.cloud.endpoint", "endpoint"), - ("yandex.cloud.iam", "iam"), - ("yandex.cloud.iot.devices", "iot-devices"), - ("yandex.cloud.k8s", "managed-kubernetes"), - ("yandex.cloud.kms", "kms"), - ("yandex.cloud.kms.v1.symmetric_crypto_service", "kms-crypto"), - ("yandex.cloud.loadbalancer", "load-balancer"), - ("yandex.cloud.loadtesting", "loadtesting"), - ("yandex.cloud.lockbox.v1.payload_service", "lockbox-payload"), - ("yandex.cloud.lockbox", "lockbox"), - ("yandex.cloud.logging.v1.log_ingestion_service", "log-ingestion"), - ("yandex.cloud.logging.v1.log_reading_service", "log-reading"), - ("yandex.cloud.logging", "logging"), - ("yandex.cloud.marketplace", "marketplace"), - ("yandex.cloud.mdb.clickhouse", "managed-clickhouse"), - ("yandex.cloud.mdb.elasticsearch", "managed-elasticsearch"), - ("yandex.cloud.mdb.greenplum", "managed-greenplum"), - ("yandex.cloud.mdb.kafka", "managed-kafka"), - ("yandex.cloud.mdb.mongodb", "managed-mongodb"), - ("yandex.cloud.mdb.mysql", "managed-mysql"), - ("yandex.cloud.mdb.opensearch", "managed-opensearch"), - ("yandex.cloud.mdb.postgresql", "managed-postgresql"), - ("yandex.cloud.mdb.redis", "managed-redis"), - ("yandex.cloud.mdb.sqlserver", "managed-sqlserver"), - ("yandex.cloud.operation", "operation"), - ("yandex.cloud.organizationmanager", "organization-manager"), - ("yandex.cloud.resourcemanager", "resource-manager"), - ("yandex.cloud.serverless.apigateway", "serverless-apigateway"), - ("yandex.cloud.serverless.apigateway.websocket", "apigateway-connections"), - ("yandex.cloud.serverless.containers", "serverless-containers"), - ("yandex.cloud.serverless.functions", "serverless-functions"), - ("yandex.cloud.serverless.triggers", "serverless-triggers"), - ("yandex.cloud.storage", "storage-api"), - ("yandex.cloud.vpc", "vpc"), - ("yandex.cloud.ydb", "ydb"), -] diff --git a/contrib/python/yandexcloud/yandexcloud/_wrappers/__init__.py b/contrib/python/yandexcloud/yandexcloud/_wrappers/__init__.py deleted file mode 100644 index 2467325a9d9..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_wrappers/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import TYPE_CHECKING - -from yandexcloud._wrappers.dataproc import Dataproc, InitializationAction - -if TYPE_CHECKING: - from yandexcloud._sdk import SDK - - -class Wrappers: - def __init__(self, sdk: "SDK"): - # pylint: disable-next=invalid-name - self.Dataproc = Dataproc - self.Dataproc.sdk = sdk - # pylint: disable-next=invalid-name - self.InitializationAction = InitializationAction diff --git a/contrib/python/yandexcloud/yandexcloud/_wrappers/dataproc/__init__.py b/contrib/python/yandexcloud/yandexcloud/_wrappers/dataproc/__init__.py deleted file mode 100644 index 9f75ef1674b..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/_wrappers/dataproc/__init__.py +++ /dev/null @@ -1,748 +0,0 @@ -# -*- coding: utf-8 -*- -# pylint: disable=no-member -# mypy: ignore-errors -import logging -import random -from typing import List, NamedTuple - -from google.protobuf.field_mask_pb2 import FieldMask -from six import string_types - -import yandex.cloud.dataproc.v1.cluster_pb2 as cluster_pb -import yandex.cloud.dataproc.v1.cluster_service_pb2 as cluster_service_pb -import yandex.cloud.dataproc.v1.cluster_service_pb2_grpc as cluster_service_grpc_pb -import yandex.cloud.dataproc.v1.common_pb2 as common_pb -import yandex.cloud.dataproc.v1.job_pb2 as job_pb -import yandex.cloud.dataproc.v1.job_service_pb2 as job_service_pb -import yandex.cloud.dataproc.v1.job_service_pb2_grpc as job_service_grpc_pb -import yandex.cloud.dataproc.v1.subcluster_pb2 as subcluster_pb -import yandex.cloud.dataproc.v1.subcluster_service_pb2 as subcluster_service_pb -import yandex.cloud.dataproc.v1.subcluster_service_pb2_grpc as subcluster_service_grpc_pb - - -class InitializationAction(NamedTuple): - uri: str # Uri of the executable file - args: List[str] # Arguments to the initialization action - timeout: int # Execution timeout - - def to_grpc(self): - return cluster_pb.InitializationAction( - uri=self.uri, - args=self.args, - timeout=self.timeout, - ) - - -class Dataproc: - """ - A base hook for Yandex.Cloud Data Proc. - - :param default_folder_id: ID of the Yandex.Cloud folder that will be used by default for nodes and clusters creation - :type default_folder_id: Optional[str] - :param default_public_ssh_key: SSH public key that will be placed to created Compute nodes, providing a root shell - :type default_public_ssh_key: Optional[str] - :param logger: Logger object - :type logger: Optional[logging.Logger] - :param sdk: SDK object. Normally is being set by Wrappers constructor - :type sdk: yandexcloud.SDK - """ - - def __init__(self, default_folder_id=None, default_public_ssh_key=None, logger=None, sdk=None): - self.sdk = sdk or self.sdk - self.log = logger - if not self.log: - self.log = logging.getLogger() - self.log.addHandler(logging.NullHandler()) - self.cluster_id = None - self.subnet_id = None - self.default_folder_id = default_folder_id - self.default_public_ssh_key = default_public_ssh_key - - def create_cluster( - self, - s3_bucket=None, - folder_id=None, - cluster_name=None, - cluster_description="", - cluster_image_version=None, - ssh_public_keys=None, - subnet_id=None, - services=None, - zone="ru-central1-b", - service_account_id=None, - masternode_resource_preset=None, - masternode_disk_size=None, - masternode_disk_type=None, - datanode_resource_preset=None, - datanode_disk_size=None, - datanode_disk_type=None, - datanode_count=None, - computenode_resource_preset=None, - computenode_disk_size=None, - computenode_disk_type=None, - computenode_count=None, - computenode_max_hosts_count=None, - computenode_measurement_duration=None, - computenode_warmup_duration=None, - computenode_stabilization_duration=None, - computenode_preemptible=None, - computenode_cpu_utilization_target=None, - computenode_decommission_timeout=None, - log_group_id=None, - properties=None, - enable_ui_proxy=False, - host_group_ids=None, - security_group_ids=None, - initialization_actions=None, - labels=None, - ): - """ - Create Yandex.Cloud Data Proc cluster. - - :param s3_bucket: Yandex.Cloud S3 bucket to store cluster logs. - Jobs will not work if the bicket is not specified. - :type s3_bucket: str - :param folder_id: ID of the folder in which cluster should be created. - :type folder_id: str - :param cluster_name: Cluster name. Must be unique inside the folder. - :type folder_id: str - :param cluster_description: Cluster description. - :type cluster_description: str - :param cluster_image_version: Cluster image version. Use default. - :type cluster_image_version: str - :param ssh_public_keys: List of SSH public keys that will be deployed to created compute instances. - :type ssh_public_keys: List[str] - :param subnet_id: ID of the subnetwork. All Data Proc cluster nodes will use one subnetwork. - :type subnet_id: str - :param services: List of services that will be installed to the cluster. Possible options: - HDFS, YARN, MAPREDUCE, HIVE, TEZ, ZOOKEEPER, HBASE, SQOOP, FLUME, SPARK, SPARK, ZEPPELIN, OOZIE - :type services: List[str] - :param zone: Availability zone to create cluster in. - Currently there are ru-central1-a, ru-central1-b and ru-central1-c. - :type zone: str - :param service_account_id: Service account id for the cluster. - Service account can be created inside the folder. - :type service_account_id: str - :param masternode_resource_preset: Resources preset (CPU+RAM configuration) - for the master node of the cluster. - :type masternode_resource_preset: str - :param masternode_disk_size: Masternode storage size in GiB. - :type masternode_disk_size: int - :param masternode_disk_type: Masternode storage type. Possible options: network-ssd, network-hdd. - :type masternode_disk_type: str - :param datanode_resource_preset: Resources preset (CPU+RAM configuration) - for the data nodes of the cluster. - :type datanode_resource_preset: str - :param datanode_disk_size: Datanodes storage size in GiB. - :type datanode_disk_size: int - :param datanode_disk_type: Datanodes storage type. Possible options: network-ssd, network-hdd. - :type datanode_disk_type: str - :param computenode_resource_preset: Resources preset (CPU+RAM configuration) - for the compute nodes of the cluster. - :type datanode_count: int - :param datanode_count: Number of data nodes. - :type computenode_resource_preset: str - :param computenode_disk_size: Computenodes storage size in GiB. - :type computenode_disk_size: int - :param computenode_disk_type: Computenodes storage type. Possible options: network-ssd, network-hdd. - :type computenode_disk_type: str - :type computenode_count: int - :param computenode_count: Number of compute nodes. - :type computenode_max_count: int - :param computenode_max_count: Maximum number of nodes of compute autoscaling subcluster. - :param computenode_warmup_duration: The warmup time of the instance in seconds. During this time, - traffic is sent to the instance, but instance metrics are not collected. In seconds. - :type computenode_warmup_duration: int - :param computenode_stabilization_duration: Minimum amount of time in seconds allotted for monitoring before - Instance Groups can reduce the number of instances in the group. - During this time, the group size doesn't decrease, even if the new metric values - indicate that it should. In seconds. - :type computenode_stabilization_duration: int - :param computenode_preemptible: Preemptible instances are stopped at least once every 24 hours, - and can be stopped at any time if their resources are needed by Compute. - :type computenode_preemptible: bool - :param computenode_cpu_utilization_target: Defines an autoscaling rule - based on the average CPU utilization of the instance group. - in percents. 10-100. - :type computenode_cpu_utilization_target: int - :param computenode_decommission_timeout: Timeout to gracefully decommission nodes during downscaling. - In seconds. - :type computenode_decommission_timeout: int - :param log_group_id: Id of log group to write logs. By default logs will be sent to default log group. - To disable cloud log sending set cluster property dataproc:disable_cloud_logging = true - :type log_group_id: str - :param properties: Properties passed to main node software, in key you need to use prefix: 'hdfs:dfs.hosts' - :type properties: Dict[str, str] - :param enable_ui_proxy: Enable UI Proxy feature for forwarding Hadoop components web interfaces - Docs: https://cloud.yandex.com/en-ru/docs/data-proc/concepts/ui-proxy - :type enable_ui_proxy: Bool - :param host_group_ids: Dedicated host groups to place VMs of cluster on. - Docs: https://cloud.yandex.com/en-ru/docs/compute/concepts/dedicated-host - :type host_group_ids: List[str] - :param security_group_ids: User security groups - Docs: https://cloud.yandex.com/en-ru/docs/data-proc/concepts/network#security-groups - :type security_group_ids: List[str] - :param initialization_actions: Set of init-actions to run when cluster starts - :type initialization_actions: List[InitializationAction] - :param labels: Cluster labels as key:value pairs. No more than 64 per resource. - :type labels: Dict[str, str] - - :return: Cluster ID - :rtype: str - """ - - # pylint: disable=too-many-arguments - # pylint: disable=too-many-locals - # pylint: disable=too-many-branches - - folder_id = folder_id or self.default_folder_id - if not folder_id: - raise RuntimeError("Folder ID must be specified to create cluster.") - - if not cluster_name: - random_int = random.randint(0, 999) - cluster_name = f"dataproc-{random_int}" - - if not subnet_id: - network_id = self.sdk.helpers.find_network_id(folder_id) - subnet_id = self.sdk.helpers.find_subnet_id(folder_id, zone, network_id) - - if not service_account_id: - service_account_id = self.sdk.helpers.find_service_account_id(folder_id) - - if not ssh_public_keys: - if self.default_public_ssh_key: - ssh_public_keys = (self.default_public_ssh_key,) - else: - raise RuntimeError("Public ssh keys must be specified.") - elif isinstance(ssh_public_keys, string_types): - ssh_public_keys = [ssh_public_keys] - - gib = 1024**3 - if masternode_disk_size: - masternode_disk_size *= gib - if datanode_disk_size: - datanode_disk_size *= gib - if computenode_disk_size: - computenode_disk_size *= gib - subclusters = [ - cluster_service_pb.CreateSubclusterConfigSpec( - name="master", - role=subcluster_pb.Role.MASTERNODE, - resources=common_pb.Resources( - resource_preset_id=masternode_resource_preset, - disk_size=masternode_disk_size, - disk_type_id=masternode_disk_type, - ), - subnet_id=subnet_id, - hosts_count=1, - ) - ] - if datanode_count: - subclusters.append( - cluster_service_pb.CreateSubclusterConfigSpec( - name="data", - role=subcluster_pb.Role.DATANODE, - resources=common_pb.Resources( - resource_preset_id=datanode_resource_preset, - disk_size=datanode_disk_size, - disk_type_id=datanode_disk_type, - ), - subnet_id=subnet_id, - hosts_count=datanode_count, - ) - ) - - if computenode_count: - autoscaling_config = None - if computenode_max_hosts_count: - autoscaling_config = subcluster_pb.AutoscalingConfig( - max_hosts_count=computenode_max_hosts_count, - measurement_duration=computenode_measurement_duration, - warmup_duration=computenode_warmup_duration, - stabilization_duration=computenode_stabilization_duration, - preemptible=computenode_preemptible, - cpu_utilization_target=computenode_cpu_utilization_target, - decommission_timeout=computenode_decommission_timeout, - ) - subclusters.append( - cluster_service_pb.CreateSubclusterConfigSpec( - name="compute", - role=subcluster_pb.Role.COMPUTENODE, - resources=common_pb.Resources( - resource_preset_id=computenode_resource_preset, - disk_size=computenode_disk_size, - disk_type_id=computenode_disk_type, - ), - subnet_id=subnet_id, - hosts_count=computenode_count, - autoscaling_config=autoscaling_config, - ) - ) - - request = cluster_service_pb.CreateClusterRequest( - folder_id=folder_id, - name=cluster_name, - description=cluster_description, - config_spec=cluster_service_pb.CreateClusterConfigSpec( - version_id=cluster_image_version, - hadoop=cluster_pb.HadoopConfig( - services=services, - ssh_public_keys=ssh_public_keys, - properties=properties, - initialization_actions=( - initialization_actions and [action.to_grpc() for action in initialization_actions] - ), - ), - subclusters_spec=subclusters, - ), - zone_id=zone, - service_account_id=service_account_id, - bucket=s3_bucket, - ui_proxy=enable_ui_proxy, - host_group_ids=host_group_ids, - security_group_ids=security_group_ids, - log_group_id=log_group_id, - labels=labels, - ) - result = self.sdk.create_operation_and_get_result( - request, - service=cluster_service_grpc_pb.ClusterServiceStub, - method_name="Create", - response_type=cluster_pb.Cluster, - meta_type=cluster_service_pb.CreateClusterMetadata, - ) - self.cluster_id = result.response.id - self.subnet_id = subnet_id # pylint: disable=attribute-defined-outside-init - return result - - def create_subcluster( - self, - subcluster_type, - name, - resource_preset=None, - disk_size=None, - disk_type=None, - hosts_count=None, - subnet_id=None, - cluster_id=None, - max_hosts_count=None, - measurement_duration=None, - warmup_duration=None, - stabilization_duration=None, - preemptible=None, - cpu_utilization_target=None, - decommission_timeout=None, - ): - """ - Create subcluster to Yandex.Cloud Data Proc cluster. - - :param name: Name of the subcluster. Must be unique in the cluster - :type name: str - :param subcluster_type: Type of the subcluster. Either "data" or "compute". - :type subcluster_type: str - :param resource_preset: Resources preset (CPU+RAM configuration) for the nodes of the cluster. - :type resource_preset: str - :param disk_size: Storage size in GiB. - :type disk_size: int - :param disk_type: Storage type. Possible options: network-ssd, network-hdd. - :type disk_type: str - :param hosts_count: Number of nodes in subcluster. - :type hosts_count: int - :param subnet_id: Subnet ID of the cluster. - :type subnet_id: str - :param cluster_id: ID of the cluster. - :type cluster_id: str - :param max_hosts_count: Upper limit for total instance autoscaling compute subcluster count. - :type max_hosts_count: int - :param measurement_duration: Time in seconds allotted for averaging metrics. In seconds. - :type measurement_duration: int - :param warmup_duration: The warmup time of the instance in seconds. During this time, - traffic is sent to the instance, but instance metrics are not collected. In seconds. - :type warmup_duration: int - :param stabilization_duration: Minimum amount of time in seconds allotted for monitoring before - Instance Groups can reduce the number of instances in the group. - During this time, the group size doesn't decrease, even if the new metric values - indicate that it should. In seconds. - :type stabilization_duration: int - :param preemptible: Preemptible instances are stopped at least once every 24 hours, - and can be stopped at any time if their resources are needed by Compute. - :type preemptible: bool - :param cpu_utilization_target: Defines an autoscaling rule - based on the average CPU utilization of the instance group. - in percents. 10-100. - :type cpu_utilization_target: int - :param decommission_timeout: Timeout to gracefully decommission nodes during downscaling. In seconds. - :type decommission_timeout: int - """ - # pylint: disable=too-many-locals - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - subnet_id = subnet_id or self.subnet_id - if not subnet_id: - raise RuntimeError("Subnet ID id must be specified.") - subnet_id = subnet_id or self.subnet_id - if not subnet_id: - raise RuntimeError("Subnet ID id must be specified.") - - types = { - "compute": subcluster_pb.Role.COMPUTENODE, - "data": subcluster_pb.Role.DATANODE, - } - if disk_size: - disk_size *= 1024**3 - resources = common_pb.Resources( - resource_preset_id=resource_preset, - disk_size=disk_size, - disk_type_id=disk_type, - ) - - self.log.info("Adding subcluster to cluster %s", cluster_id) - autoscaling_config = None - if max_hosts_count: - autoscaling_config = subcluster_pb.AutoscalingConfig( - max_hosts_count=max_hosts_count, - measurement_duration=measurement_duration, - warmup_duration=warmup_duration, - stabilization_duration=stabilization_duration, - preemptible=preemptible, - cpu_utilization_target=cpu_utilization_target, - decommission_timeout=decommission_timeout, - ) - request = subcluster_service_pb.CreateSubclusterRequest( - cluster_id=cluster_id, - name=name, - role=types[subcluster_type], - resources=resources, - subnet_id=subnet_id, - hosts_count=hosts_count, - autoscaling_config=autoscaling_config, - ) - return self.sdk.create_operation_and_get_result( - request, - service=subcluster_service_grpc_pb.SubclusterServiceStub, - method_name="Create", - response_type=subcluster_pb.Subcluster, - meta_type=subcluster_service_pb.CreateSubclusterMetadata, - ) - - def update_cluster_description(self, description, cluster_id=None): - """ - Changes Yandex.Cloud Data Proc cluster description. - - :param description: Description of the cluster. - :type description: str - :param cluster_id: ID of the cluster. - :type cluster_id: str - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - - self.log.info("Updating cluster %s", cluster_id) - mask = FieldMask(paths=["description"]) - request = cluster_service_pb.UpdateClusterRequest( - cluster_id=cluster_id, - update_mask=mask, - description=description, - ) - return self.sdk.create_operation_and_get_result( - request, - service=cluster_service_grpc_pb.ClusterServiceStub, - method_name="Update", - response_type=cluster_pb.Cluster, - meta_type=cluster_service_pb.UpdateClusterMetadata, - ) - - def delete_cluster(self, cluster_id=None): - """ - Delete Yandex.Cloud Data Proc cluster. - :param cluster_id: ID of the cluster to remove. - :type cluster_id: str - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - - self.log.info("Deleting cluster %s", cluster_id) - request = cluster_service_pb.DeleteClusterRequest(cluster_id=cluster_id) - return self.sdk.create_operation_and_get_result( - request, - service=cluster_service_grpc_pb.ClusterServiceStub, - method_name="Delete", - meta_type=cluster_service_pb.DeleteClusterMetadata, - ) - - def create_hive_job( - self, - query=None, - query_file_uri=None, - script_variables=None, - continue_on_failure=False, - properties=None, - cluster_id=None, - name="Hive job", - ): - """ - Run Hive job in Yandex.Cloud Data Proc cluster. - - :param query: Hive query. - :type query: str - :param query_file_uri: URI of the script that contains Hive queries. Can be placed in HDFS or S3. - :type query_file_uri: str - :param properties: A mapping of property names to values, used to configure Hive. - :type properties: Dist[str, str] - :param script_variables: Mapping of query variable names to values. - :type script_variables: Dist[str, str] - :param continue_on_failure: Whether to continue executing queries if a query fails. - :type continue_on_failure: boole - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if ot specified. - :type cluster_id: str - :param name: Name of the job. Used for labeling. - :type name: str - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - if (query and query_file_uri) or not (query or query_file_uri): - raise RuntimeError("Either query or query_file_uri must be specified.") - self.log.info("Running Hive job. Cluster ID: %s", cluster_id) - - hive_job = job_pb.HiveJob( - query_file_uri=query_file_uri, - script_variables=script_variables, - continue_on_failure=continue_on_failure, - properties=properties, - ) - if query: - hive_job = job_pb.HiveJob( - query_list=job_pb.QueryList(queries=query.split("\n")), - script_variables=script_variables, - continue_on_failure=continue_on_failure, - properties=properties, - ) - request = job_service_pb.CreateJobRequest( - cluster_id=cluster_id, - name=name, - hive_job=hive_job, - ) - return self.sdk.create_operation_and_get_result( - request, - service=job_service_grpc_pb.JobServiceStub, - method_name="Create", - response_type=job_pb.Job, - meta_type=job_service_pb.CreateJobMetadata, - ) - - def create_mapreduce_job( - self, - main_class=None, - main_jar_file_uri=None, - jar_file_uris=None, - archive_uris=None, - file_uris=None, - args=None, - properties=None, - cluster_id=None, - name="Mapreduce job", - ): - """ - Run Mapreduce job in Yandex.Cloud Data Proc cluster. - - :param main_jar_file_uri: URI of jar file with job. - Can be placed in HDFS or S3. Can be specified instead of main_class. - :type main_class: str - :param main_class: Name of the main class of the job. Can be specified instead of main_jar_file_uri. - :type main_class: str - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :type file_uris: List[str] - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param properties: Properties for the job. - :type properties: Dist[str, str] - :param args: Arguments to be passed to the job. - :type args: List[str] - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if ot specified. - :type cluster_id: str - :param name: Name of the job. Used for labeling. - :type name: str - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - self.log.info("Running Mapreduce job. Cluster ID: %s", cluster_id) - - request = job_service_pb.CreateJobRequest( - cluster_id=cluster_id, - name=name, - mapreduce_job=job_pb.MapreduceJob( - main_class=main_class, - main_jar_file_uri=main_jar_file_uri, - jar_file_uris=jar_file_uris, - archive_uris=archive_uris, - file_uris=file_uris, - args=args, - properties=properties, - ), - ) - return self.sdk.create_operation_and_get_result( - request, - service=job_service_grpc_pb.JobServiceStub, - method_name="Create", - response_type=job_pb.Job, - meta_type=job_service_pb.CreateJobMetadata, - ) - - def create_spark_job( - self, - main_jar_file_uri=None, - main_class=None, - file_uris=None, - archive_uris=None, - jar_file_uris=None, - args=None, - properties=None, - cluster_id=None, - name="Spark job", - packages=None, - repositories=None, - exclude_packages=None, - ): - """ - Run Spark job in Yandex.Cloud Data Proc cluster. - - :param main_jar_file_uri: URI of jar file with job. Can be placed in HDFS or S3. - :type main_class: str - :param main_class: Name of the main class of the job. - :type main_class: str - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :type file_uris: List[str] - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param properties: Properties for the job. - :type properties: Dist[str, str] - :param args: Arguments to be passed to the job. - :type args: List[str] - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if ot specified. - :type cluster_id: str - :param name: Name of the job. Used for labeling. - :type name: str - :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. - :type packages: List[str] - :param repositories: List of additional remote repositories to search for the maven - coordinates given with --packages. - :type repositories: List[str] - :param exclude_packages: List of groupId:artifactId, to exclude while resolving the - dependencies provided in --packages to avoid dependency conflicts. - :type exclude_packages: List[str] - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - self.log.info("Running Spark job. Cluster ID: %s", cluster_id) - - request = job_service_pb.CreateJobRequest( - cluster_id=cluster_id, - name=name, - spark_job=job_pb.SparkJob( - main_jar_file_uri=main_jar_file_uri, - main_class=main_class, - file_uris=file_uris, - archive_uris=archive_uris, - jar_file_uris=jar_file_uris, - args=args, - properties=properties, - packages=packages, - repositories=repositories, - exclude_packages=exclude_packages, - ), - ) - return self.sdk.create_operation_and_get_result( - request, - service=job_service_grpc_pb.JobServiceStub, - method_name="Create", - response_type=job_pb.Job, - meta_type=job_service_pb.CreateJobMetadata, - ) - - def create_pyspark_job( - self, - main_python_file_uri=None, - python_file_uris=None, - file_uris=None, - archive_uris=None, - jar_file_uris=None, - args=None, - properties=None, - cluster_id=None, - name="Pyspark job", - packages=None, - repositories=None, - exclude_packages=None, - ): - """ - Run Pyspark job in Yandex.Cloud Data Proc cluster. - - :param main_python_file_uri: URI of python file with job. Can be placed in HDFS or S3. - :type main_python_file_uri: str - :param python_file_uris: URIs of python files used in the job. Can be placed in HDFS or S3. - :type python_file_uris: List[str] - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :type file_uris: List[str] - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :type archive_uris: List[str] - :param properties: Properties for the job. - :type properties: Dist[str, str] - :param args: Arguments to be passed to the job. - :type args: List[str] - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if ot specified. - :type cluster_id: str - :param name: Name of the job. Used for labeling. - :type name: str - :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. - :type packages: List[str] - :param repositories: List of additional remote repositories to search for the maven - coordinates given with --packages. - :type repositories: List[str] - :param exclude_packages: List of groupId:artifactId, to exclude while resolving the - dependencies provided in --packages to avoid dependency conflicts. - :type exclude_packages: List[str] - """ - cluster_id = cluster_id or self.cluster_id - if not cluster_id: - raise RuntimeError("Cluster id must be specified.") - self.log.info("Running Pyspark job. Cluster ID: %s", cluster_id) - request = job_service_pb.CreateJobRequest( - cluster_id=cluster_id, - name=name, - pyspark_job=job_pb.PysparkJob( - main_python_file_uri=main_python_file_uri, - python_file_uris=python_file_uris, - file_uris=file_uris, - archive_uris=archive_uris, - jar_file_uris=jar_file_uris, - args=args, - properties=properties, - packages=packages, - repositories=repositories, - exclude_packages=exclude_packages, - ), - ) - return self.sdk.create_operation_and_get_result( - request, - service=job_service_grpc_pb.JobServiceStub, - method_name="Create", - response_type=job_pb.Job, - meta_type=job_service_pb.CreateJobMetadata, - ) diff --git a/contrib/python/yandexcloud/yandexcloud/auth.py b/contrib/python/yandexcloud/yandexcloud/auth.py deleted file mode 100644 index 079a427f4bc..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/auth.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Dict, Optional - -from yandex.cloud.iam.v1.iam_token_service_pb2_grpc import IamTokenServiceStub -from yandexcloud._auth_fabric import ( - YC_API_ENDPOINT, - IamTokenAuth, - MetadataAuth, - get_auth_token_requester, -) -from yandexcloud._sdk import SDK - - -def get_auth_token( - token: Optional[str] = None, - service_account_key: Optional[Dict[str, str]] = None, - iam_token: Optional[str] = None, - metadata_addr: Optional[str] = None, - endpoint: Optional[str] = None, -) -> str: - if endpoint is None: - endpoint = YC_API_ENDPOINT - requester = get_auth_token_requester( - token=token, - service_account_key=service_account_key, - iam_token=iam_token, - metadata_addr=metadata_addr, - endpoint=endpoint, - ) - if isinstance(requester, (MetadataAuth, IamTokenAuth)): - return requester.get_token() - - sdk = SDK() - client = sdk.client(IamTokenServiceStub) - return client.Create(requester.get_token_request()).iam_token diff --git a/contrib/python/yandexcloud/yandexcloud/operations.py b/contrib/python/yandexcloud/yandexcloud/operations.py deleted file mode 100644 index 2e7a615b0cc..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/operations.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - import google.protobuf.message - - from yandex.cloud.operation.operation_pb2 import Operation - - # from yandex.cloud.api.operation_pb2 import Operation - - -class OperationResult: - def __init__( - self, - operation: "Operation", - response: Optional["google.protobuf.message.Message"] = None, - meta: Optional["google.protobuf.message.Message"] = None, - ): - self.operation = operation - self.response = response - self.meta = meta - - -class OperationError(RuntimeError): - def __init__(self, message: str, operation_result: OperationResult): - super(OperationError, self).__init__(message) # pylint: disable=super-with-arguments - self.message = message - self.operation_result = operation_result diff --git a/contrib/python/yandexcloud/yandexcloud/py.typed b/contrib/python/yandexcloud/yandexcloud/py.typed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/contrib/python/yandexcloud/yandexcloud/py.typed +++ /dev/null |