aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkomels <komels@yandex-team.ru>2022-04-15 16:53:39 +0300
committerkomels <komels@yandex-team.ru>2022-04-15 16:53:39 +0300
commit703a2fb6e100d202d1c7fcd052d73bd5affef408 (patch)
tree22b7320c06bb04d86dbf7b9af9ae44281331cd15
parent3375bbfda1e2afb03aa2072bf5f2f2c3a26026e8 (diff)
downloadydb-703a2fb6e100d202d1c7fcd052d73bd5affef408.tar.gz
Move 'kikimr/yndx'-depending tests out of ydb/core
ref:0a380e13308d579e0545a76924330d1ca5129c43
-rw-r--r--CMakeLists.darwin.txt40
-rw-r--r--CMakeLists.linux.txt40
-rw-r--r--cloud/README.md7
-rw-r--r--cloud/__init__.py0
-rw-r--r--cloud/bitbucket/common-api/yandex/cloud/api/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/common-api/yandex/cloud/api/operation.proto24
-rw-r--r--cloud/bitbucket/common-api/yandex/cloud/api/tools/CMakeLists.txt51
-rw-r--r--cloud/bitbucket/common-api/yandex/cloud/api/tools/options.proto87
-rw-r--r--cloud/bitbucket/private-api/README.md52
-rw-r--r--cloud/bitbucket/private-api/README.roles.md108
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/CMakeLists.txt52
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/access/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/access/access.proto94
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/restriction.proto58
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/CMakeLists.txt102
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/access_binding_service.proto258
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key.proto15
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key_service.proto104
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto18
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key_service.proto117
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/access_binding_service.proto29
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/permission_service.proto49
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/compute/os_login_service.proto86
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/access_binding_service.proto60
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/key_service.proto28
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/membership_service.proto39
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/os_login_service.proto23
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/role_service.proto46
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/service_account_service.proto58
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/user_account_service.proto29
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/gizmo_service.proto34
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_cookie_service.proto56
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.proto69
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key.proto32
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key_service.proto140
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/membership_service.proto50
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/console/totp_profile_service.proto32
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile.proto43
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile_service.proto243
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/operation_service.proto17
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile.proto56
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile_service.proto125
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client.proto18
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client_service.proto99
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope.proto13
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope_service.proto90
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/openapi-meta.yaml6
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/operation_service.proto20
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login.proto12
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login_service.proto45
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission.proto11
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_service.proto80
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage.proto36
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage_service.proto70
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permissions.yaml1481
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quota_service.proto17
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quotas.yaml2
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type.proto11
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type_service.proto105
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resources.yaml26
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction.proto16
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_service.proto89
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type.proto32
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type_service.proto114
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role.proto13
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role_service.proto95
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/roles.yaml568
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/root_service.proto34
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate.proto39
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate_service.proto110
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation.proto92
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation_service.proto171
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/scopes.yaml107
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account.proto20
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account_service.proto146
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key.proto15
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key_service.proto86
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/subject_service.proto81
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/iam_token.proto14
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token_agent.proto26
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/agreement_service.proto36
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/auth_service.proto25
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/health_check_service.proto29
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/passport_federation_service.proto45
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/policy_service.proto79
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/roles_service.proto28
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/service_account_access_key_service.proto54
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/user_service.proto90
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto25
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account.proto46
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account_service.proto106
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto32
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_user_account_service.proto52
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/claims.proto90
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/CMakeLists.txt59
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/claim_service.proto25
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/clients.yaml72
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/cloud_user.proto23
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/oauth_request.proto15
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/permissions.yaml21
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/roles.yaml42
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/scopes.yaml39
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/session_service.proto153
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/operation/CMakeLists.txt51
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/operation/operation.proto28
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/quota/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/quota/quota.proto63
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/resources.yaml3
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/restrictions.yaml32
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/roles.yaml447
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/sensitive.proto24
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/CMakeLists.txt54
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/access_service.proto255
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/resource.proto19
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/services.yaml186
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/stages.yaml173
-rw-r--r--cloud/bitbucket/private-api/yandex/cloud/priv/validation.proto29
-rw-r--r--cloud/bitbucket/public-api/AUTHORS32
-rw-r--r--cloud/bitbucket/public-api/LICENSE21
-rw-r--r--cloud/bitbucket/public-api/README.md15
-rw-r--r--cloud/bitbucket/public-api/internal-CONTRIBUTING.md105
-rw-r--r--cloud/bitbucket/public-api/openapi-descriptions.yaml25
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/CMakeLists.txt52
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/access/CMakeLists.txt53
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/access/access.proto120
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/CMakeLists.txt69
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key.proto23
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key_service.proto167
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key.proto29
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key_service.proto168
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/openapi-meta.yaml7
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.proto52
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/key.proto44
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/key_service.proto187
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/openapi-meta.yaml7
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/role.proto15
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/role_service.proto61
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate.proto32
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate_service.proto186
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation.proto83
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation_service.proto317
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/openapi-meta.yaml7
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account.proto30
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account_service.proto215
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account.proto47
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account_service.proto23
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/iam/v1/yandex_passport_user_account_service.proto23
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/imports.proto23
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/operation/CMakeLists.txt56
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/operation/openapi-meta.yaml7
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/operation/operation.proto55
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/operation/operation_service.proto37
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/validation.proto29
-rw-r--r--cloud/bitbucket/public-api/yandex/cloud/validation.yadoc.yaml127
-rw-r--r--contrib/libs/googleapis-common-protos/.yandex_meta/devtools.copyrights.report192
-rw-r--r--contrib/libs/googleapis-common-protos/.yandex_meta/devtools.licenses.report128
-rw-r--r--contrib/libs/googleapis-common-protos/.yandex_meta/licenses.list.txt259
-rw-r--r--contrib/libs/googleapis-common-protos/CHANGELOG.md65
-rw-r--r--contrib/libs/googleapis-common-protos/CMakeLists.txt100
-rw-r--r--contrib/libs/googleapis-common-protos/CODE_OF_CONDUCT.md43
-rw-r--r--contrib/libs/googleapis-common-protos/CONTRIBUTING.md29
-rw-r--r--contrib/libs/googleapis-common-protos/LICENSE202
-rw-r--r--contrib/libs/googleapis-common-protos/README.md8
-rw-r--r--contrib/libs/googleapis-common-protos/SECURITY.md7
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/README.md46
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/annotations.proto31
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/auth.proto181
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/backend.proto51
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/billing.proto67
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/client.proto99
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/config_change.proto85
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/consumer.proto83
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/context.proto63
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/control.proto33
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/distribution.proto213
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/documentation.proto157
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/endpoint.proto71
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/field_behavior.proto84
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/http.proto318
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/httpbody.proto76
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/label.proto49
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/launch_stage.proto67
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/log.proto55
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/logging.proto83
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/metric.proto192
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/monitored_resource.proto116
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/monitoring.proto89
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/quota.proto259
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/resource.proto299
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/routing.proto461
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/service.proto175
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/source_info.proto32
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/system_parameter.proto96
-rw-r--r--contrib/libs/googleapis-common-protos/google/api/usage.proto92
-rw-r--r--contrib/libs/googleapis-common-protos/google/cloud/extended_operations.proto150
-rw-r--r--contrib/libs/googleapis-common-protos/google/cloud/location/locations.proto108
-rw-r--r--contrib/libs/googleapis-common-protos/google/gapic/metadata/gapic_metadata.proto92
-rw-r--r--contrib/libs/googleapis-common-protos/google/logging/type/README.md12
-rw-r--r--contrib/libs/googleapis-common-protos/google/logging/type/http_request.proto92
-rw-r--r--contrib/libs/googleapis-common-protos/google/logging/type/log_severity.proto72
-rw-r--r--contrib/libs/googleapis-common-protos/google/longrunning/README.md31
-rw-r--r--contrib/libs/googleapis-common-protos/google/longrunning/operations.proto247
-rw-r--r--contrib/libs/googleapis-common-protos/google/rpc/README.md18
-rw-r--r--contrib/libs/googleapis-common-protos/google/rpc/code.proto186
-rw-r--r--contrib/libs/googleapis-common-protos/google/rpc/context/attribute_context.proto287
-rw-r--r--contrib/libs/googleapis-common-protos/google/rpc/error_details.proto246
-rw-r--r--contrib/libs/googleapis-common-protos/google/rpc/status.proto47
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/README.md7
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/calendar_period.proto57
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/color.proto170
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/date.proto50
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/datetime.proto97
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/dayofweek.proto51
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/expr.proto51
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/fraction.proto34
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/latlng.proto37
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/money.proto43
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/month.proto66
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/postal_address.proto135
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/quaternion.proto95
-rw-r--r--contrib/libs/googleapis-common-protos/google/type/timeofday.proto44
-rw-r--r--contrib/libs/minilzo/.yandex_meta/devtools.copyrights.report59
-rw-r--r--contrib/libs/minilzo/.yandex_meta/devtools.licenses.report45
-rw-r--r--contrib/libs/minilzo/.yandex_meta/licenses.list.txt26
-rw-r--r--contrib/libs/minilzo/CMakeLists.txt20
-rw-r--r--contrib/libs/minilzo/lzo_crc.c154
-rw-r--r--contrib/libs/minilzo/lzoconf.h453
-rw-r--r--contrib/libs/minilzo/lzodefs.h3134
-rw-r--r--contrib/libs/minilzo/minilzo.c6234
-rw-r--r--contrib/libs/minilzo/minilzo.h109
-rw-r--r--contrib/libs/minilzo/protect.h18
-rw-r--r--kikimr/.gitignore17
-rw-r--r--kikimr/.kikimr.root1
-rw-r--r--kikimr/README.md40
-rw-r--r--kikimr/a.yaml532
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt60
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h60
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h108
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp119
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h24
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h48
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp22
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h87
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h39
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp113
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h63
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h39
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp421
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h50
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h156
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp256
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h66
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp155
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h64
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp243
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp612
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h121
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp90
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp349
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp290
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h77
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp265
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp20
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h51
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp31
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h50
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h105
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp20
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h35
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp20
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h58
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h303
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp40
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp645
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h93
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp227
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp423
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h147
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp360
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp540
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h316
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp8
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp358
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h118
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp133
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp506
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h107
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp202
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp31
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h23
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp26
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp526
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h107
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp604
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp370
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h75
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp355
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp96
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h88
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp75
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp86
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp88
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp24
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h12
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp46
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp561
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h75
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h61
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h35
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h42
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h50
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h106
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp236
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp174
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/types.h402
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt30
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp263
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h117
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h323
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp95
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h43
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp35
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h129
-rw-r--r--kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h101
-rw-r--r--kikimr/public/README.md6
-rw-r--r--kikimr/public/sdk/cpp/README.md3
-rw-r--r--kikimr/public/sdk/cpp/client/CHANGELOG.md60
-rw-r--r--kikimr/public/sdk/cpp/client/iam/CMakeLists.txt25
-rw-r--r--kikimr/public/sdk/cpp/client/iam/iam.cpp361
-rw-r--r--kikimr/public/sdk/cpp/client/iam/iam.h50
-rw-r--r--kikimr/public/sdk/cpp/client/ydb_persqueue/CMakeLists.txt16
-rw-r--r--kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/CMakeLists.txt22
-rw-r--r--kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.cpp82
-rw-r--r--kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.h18
-rw-r--r--kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h2
-rw-r--r--kikimr/yndx/api/grpc/CMakeLists.txt43
-rw-r--r--kikimr/yndx/api/grpc/persqueue.proto68
-rw-r--r--kikimr/yndx/api/grpc/ydb_yndx_keyvalue_v1.proto43
-rw-r--r--kikimr/yndx/api/grpc/ydb_yndx_rate_limiter_v1.proto35
-rw-r--r--kikimr/yndx/api/protos/CMakeLists.txt40
-rw-r--r--kikimr/yndx/api/protos/persqueue.proto335
-rw-r--r--kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto460
-rw-r--r--kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto273
-rw-r--r--kikimr/yndx/grpc_services/persqueue/CMakeLists.txt38
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_actor.h928
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.cpp86
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.h77
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_read.cpp268
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_read.h146
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_read_actor.cpp2585
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_session.h317
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_write.cpp221
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_write.h148
-rw-r--r--kikimr/yndx/grpc_services/persqueue/grpc_pq_write_actor.cpp1055
-rw-r--r--kikimr/yndx/grpc_services/persqueue/persqueue.cpp59
-rw-r--r--kikimr/yndx/grpc_services/persqueue/persqueue.h49
-rw-r--r--kikimr/yndx/grpc_services/persqueue/persqueue_compat_ut.cpp122
-rw-r--r--kikimr/yndx/grpc_services/persqueue/persqueue_ut.cpp2405
-rw-r--r--kikimr/yndx/grpc_services/persqueue/protocol_compatibility_ut.cpp80
-rw-r--r--kikimr/yndx/grpc_services/persqueue/ut/definitions.h18
-rw-r--r--kikimr/yndx/persqueue/msgbus_server/CMakeLists.txt19
-rw-r--r--kikimr/yndx/persqueue/msgbus_server/read_session_info.cpp16
-rw-r--r--kikimr/yndx/persqueue/msgbus_server/read_session_info.h43
-rw-r--r--kikimr/yndx/persqueue/read_batch_converter/CMakeLists.txt18
-rw-r--r--kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.cpp43
-rw-r--r--kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h10
-rw-r--r--library/cpp/grpc/common/CMakeLists.txt15
-rw-r--r--library/cpp/grpc/common/time_point.h23
-rw-r--r--library/cpp/http/simple/.arcignore1
-rw-r--r--library/cpp/http/simple/CMakeLists.txt21
-rw-r--r--library/cpp/http/simple/http_client.cpp356
-rw-r--r--library/cpp/http/simple/http_client.h276
-rw-r--r--library/cpp/http/simple/http_client_options.h59
-rw-r--r--library/cpp/http/simple/ut/http_ut.cpp439
-rw-r--r--library/cpp/http/simple/ut/https_server/http_server.crt19
-rw-r--r--library/cpp/http/simple/ut/https_server/http_server.key28
-rw-r--r--library/cpp/http/simple/ut/https_server/main.go70
-rw-r--r--library/cpp/http/simple/ut/https_ut.cpp97
-rw-r--r--library/cpp/openssl/big_integer/CMakeLists.txt19
-rw-r--r--library/cpp/openssl/big_integer/big_integer.cpp61
-rw-r--r--library/cpp/openssl/big_integer/big_integer.h57
-rw-r--r--library/cpp/openssl/big_integer/ut/big_integer_ut.cpp43
-rw-r--r--library/cpp/openssl/crypto/CMakeLists.txt22
-rw-r--r--library/cpp/openssl/crypto/rsa.cpp56
-rw-r--r--library/cpp/openssl/crypto/rsa.h34
-rw-r--r--library/cpp/openssl/crypto/sha.cpp62
-rw-r--r--library/cpp/openssl/crypto/sha.h78
-rw-r--r--library/cpp/openssl/crypto/sha_ut.cpp62
-rw-r--r--library/cpp/openssl/crypto/ut/rsa_ut.cpp28
-rw-r--r--library/cpp/streams/lzop/CMakeLists.txt18
-rw-r--r--library/cpp/streams/lzop/lzop.cpp827
-rw-r--r--library/cpp/streams/lzop/lzop.h34
-rw-r--r--library/cpp/streams/lzop/lzop_ut.cpp54
-rw-r--r--library/cpp/string_utils/secret_string/CMakeLists.txt18
-rw-r--r--library/cpp/string_utils/secret_string/secret_string.cpp68
-rw-r--r--library/cpp/string_utils/secret_string/secret_string.h74
-rw-r--r--library/cpp/string_utils/secret_string/ut/secret_string_ut.cpp147
-rw-r--r--library/cpp/threading/atomic/CMakeLists.txt17
-rw-r--r--library/cpp/tvmauth/CMakeLists.txt64
-rw-r--r--library/cpp/tvmauth/README.md2
-rw-r--r--library/cpp/tvmauth/a.yaml29
-rw-r--r--library/cpp/tvmauth/checked_service_ticket.h71
-rw-r--r--library/cpp/tvmauth/checked_user_ticket.h91
-rw-r--r--library/cpp/tvmauth/client/CMakeLists.txt61
-rw-r--r--library/cpp/tvmauth/client/README.md84
-rw-r--r--library/cpp/tvmauth/client/client_status.cpp6
-rw-r--r--library/cpp/tvmauth/client/client_status.h82
-rw-r--r--library/cpp/tvmauth/client/examples/create_with_tvmapi/create.cpp102
-rw-r--r--library/cpp/tvmauth/client/examples/create_with_tvmtool/create.cpp34
-rw-r--r--library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.cpp84
-rw-r--r--library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.h35
-rw-r--r--library/cpp/tvmauth/client/exception.h23
-rw-r--r--library/cpp/tvmauth/client/facade.cpp131
-rw-r--r--library/cpp/tvmauth/client/facade.h119
-rw-r--r--library/cpp/tvmauth/client/logger.cpp12
-rw-r--r--library/cpp/tvmauth/client/logger.h59
-rw-r--r--library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.cpp126
-rw-r--r--library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.h60
-rw-r--r--library/cpp/tvmauth/client/misc/api/dynamic_dst/ut/tvm_client_ut.cpp635
-rw-r--r--library/cpp/tvmauth/client/misc/api/retry_settings.h33
-rw-r--r--library/cpp/tvmauth/client/misc/api/roles_fetcher.cpp164
-rw-r--r--library/cpp/tvmauth/client/misc/api/roles_fetcher.h63
-rw-r--r--library/cpp/tvmauth/client/misc/api/settings.cpp89
-rw-r--r--library/cpp/tvmauth/client/misc/api/settings.h302
-rw-r--r--library/cpp/tvmauth/client/misc/api/threaded_updater.cpp954
-rw-r--r--library/cpp/tvmauth/client/misc/api/threaded_updater.h140
-rw-r--r--library/cpp/tvmauth/client/misc/async_updater.cpp152
-rw-r--r--library/cpp/tvmauth/client/misc/async_updater.h183
-rw-r--r--library/cpp/tvmauth/client/misc/checker.h67
-rw-r--r--library/cpp/tvmauth/client/misc/default_uid_checker.h46
-rw-r--r--library/cpp/tvmauth/client/misc/disk_cache.cpp165
-rw-r--r--library/cpp/tvmauth/client/misc/disk_cache.h50
-rw-r--r--library/cpp/tvmauth/client/misc/exponential_backoff.h94
-rw-r--r--library/cpp/tvmauth/client/misc/fetch_result.h13
-rw-r--r--library/cpp/tvmauth/client/misc/getter.h66
-rw-r--r--library/cpp/tvmauth/client/misc/last_error.cpp115
-rw-r--r--library/cpp/tvmauth/client/misc/last_error.h51
-rw-r--r--library/cpp/tvmauth/client/misc/proc_info.cpp53
-rw-r--r--library/cpp/tvmauth/client/misc/proc_info.h18
-rw-r--r--library/cpp/tvmauth/client/misc/retry_settings/v1/CMakeLists.txt31
-rw-r--r--library/cpp/tvmauth/client/misc/retry_settings/v1/settings.proto21
-rw-r--r--library/cpp/tvmauth/client/misc/roles/decoder.cpp93
-rw-r--r--library/cpp/tvmauth/client/misc/roles/decoder.h32
-rw-r--r--library/cpp/tvmauth/client/misc/roles/entities_index.cpp114
-rw-r--r--library/cpp/tvmauth/client/misc/roles/entities_index.h107
-rw-r--r--library/cpp/tvmauth/client/misc/roles/parser.cpp149
-rw-r--r--library/cpp/tvmauth/client/misc/roles/parser.h36
-rw-r--r--library/cpp/tvmauth/client/misc/roles/roles.cpp101
-rw-r--r--library/cpp/tvmauth/client/misc/roles/roles.h186
-rw-r--r--library/cpp/tvmauth/client/misc/roles/types.h70
-rw-r--r--library/cpp/tvmauth/client/misc/settings.h13
-rw-r--r--library/cpp/tvmauth/client/misc/src_checker.h46
-rw-r--r--library/cpp/tvmauth/client/misc/threaded_updater.cpp111
-rw-r--r--library/cpp/tvmauth/client/misc/threaded_updater.h76
-rw-r--r--library/cpp/tvmauth/client/misc/tool/meta_info.cpp208
-rw-r--r--library/cpp/tvmauth/client/misc/tool/meta_info.h69
-rw-r--r--library/cpp/tvmauth/client/misc/tool/roles_fetcher.cpp81
-rw-r--r--library/cpp/tvmauth/client/misc/tool/roles_fetcher.h49
-rw-r--r--library/cpp/tvmauth/client/misc/tool/settings.cpp37
-rw-r--r--library/cpp/tvmauth/client/misc/tool/settings.h137
-rw-r--r--library/cpp/tvmauth/client/misc/tool/threaded_updater.cpp370
-rw-r--r--library/cpp/tvmauth/client/misc/tool/threaded_updater.h58
-rw-r--r--library/cpp/tvmauth/client/misc/utils.cpp46
-rw-r--r--library/cpp/tvmauth/client/misc/utils.h95
-rw-r--r--library/cpp/tvmauth/client/mocked_updater.cpp60
-rw-r--r--library/cpp/tvmauth/client/mocked_updater.h43
-rw-r--r--library/cpp/tvmauth/client/ut/async_updater_ut.cpp165
-rw-r--r--library/cpp/tvmauth/client/ut/checker_ut.cpp176
-rw-r--r--library/cpp/tvmauth/client/ut/client_status_ut.cpp18
-rw-r--r--library/cpp/tvmauth/client/ut/common.h240
-rw-r--r--library/cpp/tvmauth/client/ut/default_uid_checker_ut.cpp52
-rw-r--r--library/cpp/tvmauth/client/ut/disk_cache_ut.cpp204
-rw-r--r--library/cpp/tvmauth/client/ut/exponential_backoff_ut.cpp44
-rw-r--r--library/cpp/tvmauth/client/ut/facade_ut.cpp167
-rw-r--r--library/cpp/tvmauth/client/ut/files/ok.cachebin113 -> 0 bytes
-rw-r--r--library/cpp/tvmauth/client/ut/files/public_keysbin2840 -> 0 bytes
-rw-r--r--library/cpp/tvmauth/client/ut/files/rolesbin295 -> 0 bytes
-rw-r--r--library/cpp/tvmauth/client/ut/files/service_ticketsbin250 -> 0 bytes
-rw-r--r--library/cpp/tvmauth/client/ut/last_error_ut.cpp56
-rw-r--r--library/cpp/tvmauth/client/ut/logger_ut.cpp43
-rw-r--r--library/cpp/tvmauth/client/ut/roles/decoder_ut.cpp163
-rw-r--r--library/cpp/tvmauth/client/ut/roles/entities_index_ut.cpp358
-rw-r--r--library/cpp/tvmauth/client/ut/roles/parser_ut.cpp161
-rw-r--r--library/cpp/tvmauth/client/ut/roles/roles_ut.cpp419
-rw-r--r--library/cpp/tvmauth/client/ut/roles/tvmapi_roles_fetcher_ut.cpp197
-rw-r--r--library/cpp/tvmauth/client/ut/roles/tvmtool_roles_fetcher_ut.cpp103
-rw-r--r--library/cpp/tvmauth/client/ut/settings_ut.cpp169
-rw-r--r--library/cpp/tvmauth/client/ut/src_checker_ut.cpp47
-rw-r--r--library/cpp/tvmauth/client/ut/tvmapi_updater_ut.cpp1272
-rw-r--r--library/cpp/tvmauth/client/ut/tvmtool_updater_ut.cpp756
-rw-r--r--library/cpp/tvmauth/client/ut/utils_ut.cpp88
-rw-r--r--library/cpp/tvmauth/deprecated/README.md2
-rw-r--r--library/cpp/tvmauth/deprecated/service_context.cpp37
-rw-r--r--library/cpp/tvmauth/deprecated/service_context.h63
-rw-r--r--library/cpp/tvmauth/deprecated/user_context.cpp20
-rw-r--r--library/cpp/tvmauth/deprecated/user_context.h30
-rw-r--r--library/cpp/tvmauth/exception.h20
-rw-r--r--library/cpp/tvmauth/src/parser.cpp97
-rw-r--r--library/cpp/tvmauth/src/parser.h51
-rw-r--r--library/cpp/tvmauth/src/protos/CMakeLists.txt32
-rw-r--r--library/cpp/tvmauth/src/protos/ticket2.proto31
-rw-r--r--library/cpp/tvmauth/src/protos/tvm_keys.proto36
-rw-r--r--library/cpp/tvmauth/src/rw/CMakeLists.txt27
-rw-r--r--library/cpp/tvmauth/src/rw/keys.cpp138
-rw-r--r--library/cpp/tvmauth/src/rw/keys.h65
-rw-r--r--library/cpp/tvmauth/src/rw/rw.h86
-rw-r--r--library/cpp/tvmauth/src/rw/rw_asn1.c81
-rw-r--r--library/cpp/tvmauth/src/rw/rw_key.c135
-rw-r--r--library/cpp/tvmauth/src/rw/rw_lib.c77
-rw-r--r--library/cpp/tvmauth/src/rw/rw_ossl.c473
-rw-r--r--library/cpp/tvmauth/src/rw/rw_pss.c328
-rw-r--r--library/cpp/tvmauth/src/rw/rw_pss_sign.c211
-rw-r--r--library/cpp/tvmauth/src/rw/rw_sign.c46
-rw-r--r--library/cpp/tvmauth/src/rw/ut/rw_ut.cpp200
-rw-r--r--library/cpp/tvmauth/src/rw/ut_large/gen/main.cpp32
-rw-r--r--library/cpp/tvmauth/src/rw/ut_large/test.py35
-rw-r--r--library/cpp/tvmauth/src/service_impl.cpp203
-rw-r--r--library/cpp/tvmauth/src/service_impl.h77
-rw-r--r--library/cpp/tvmauth/src/service_ticket.cpp41
-rw-r--r--library/cpp/tvmauth/src/status.cpp32
-rw-r--r--library/cpp/tvmauth/src/unittest.cpp14
-rw-r--r--library/cpp/tvmauth/src/user_impl.cpp241
-rw-r--r--library/cpp/tvmauth/src/user_impl.h72
-rw-r--r--library/cpp/tvmauth/src/user_ticket.cpp56
-rw-r--r--library/cpp/tvmauth/src/ut/parser_ut.cpp143
-rw-r--r--library/cpp/tvmauth/src/ut/public_ut.cpp290
-rw-r--r--library/cpp/tvmauth/src/ut/service_ut.cpp156
-rw-r--r--library/cpp/tvmauth/src/ut/user_ut.cpp216
-rw-r--r--library/cpp/tvmauth/src/ut/utils_ut.cpp95
-rw-r--r--library/cpp/tvmauth/src/ut/version_ut.cpp18
-rw-r--r--library/cpp/tvmauth/src/utils.cpp162
-rw-r--r--library/cpp/tvmauth/src/utils.h30
-rw-r--r--library/cpp/tvmauth/src/version1
-rw-r--r--library/cpp/tvmauth/src/version.cpp26
-rw-r--r--library/cpp/tvmauth/ticket_status.h23
-rw-r--r--library/cpp/tvmauth/type.h11
-rw-r--r--library/cpp/tvmauth/unittest.h20
-rw-r--r--library/cpp/tvmauth/utils.cpp18
-rw-r--r--library/cpp/tvmauth/utils.h12
-rw-r--r--library/cpp/tvmauth/version.h7
-rw-r--r--ydb/core/client/server/pq_metacache_v2_ut.cpp355
-rw-r--r--ydb/core/client/server/ut/CMakeLists.darwin.txt2
-rw-r--r--ydb/core/client/server/ut/CMakeLists.linux.txt2
546 files changed, 0 insertions, 75161 deletions
diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt
index 3d9b98e74b..6e09af09b6 100644
--- a/CMakeLists.darwin.txt
+++ b/CMakeLists.darwin.txt
@@ -1151,46 +1151,6 @@ add_subdirectory(ydb/core/client/metadata/ut)
add_subdirectory(ydb/core/client/minikql_compile/ut)
add_subdirectory(ydb/core/client/minikql_result_lib/ut)
add_subdirectory(ydb/core/client/server/ut)
-add_subdirectory(kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils)
-add_subdirectory(kikimr/persqueue/sdk/deprecated/cpp/v2)
-add_subdirectory(kikimr/public/sdk/cpp/client/iam)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/iam/v1)
-add_subdirectory(contrib/libs/googleapis-common-protos)
-add_subdirectory(cloud/bitbucket/common-api/yandex/cloud/api)
-add_subdirectory(cloud/bitbucket/common-api/yandex/cloud/api/tools)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/access)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/operation)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/access)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/oauth)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/operation)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/quota)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1)
-add_subdirectory(library/cpp/http/simple)
-add_subdirectory(library/cpp/threading/atomic)
-add_subdirectory(kikimr/public/sdk/cpp/client/ydb_persqueue)
-add_subdirectory(kikimr/public/sdk/cpp/client/ydb_persqueue/codecs)
-add_subdirectory(library/cpp/streams/lzop)
-add_subdirectory(contrib/libs/minilzo)
-add_subdirectory(kikimr/yndx/api/grpc)
-add_subdirectory(kikimr/yndx/api/protos)
-add_subdirectory(kikimr/yndx/persqueue/read_batch_converter)
-add_subdirectory(library/cpp/grpc/common)
-add_subdirectory(library/cpp/tvmauth/client)
-add_subdirectory(library/cpp/openssl/crypto)
-add_subdirectory(library/cpp/openssl/big_integer)
-add_subdirectory(library/cpp/tvmauth)
-add_subdirectory(library/cpp/string_utils/secret_string)
-add_subdirectory(library/cpp/tvmauth/src/protos)
-add_subdirectory(library/cpp/tvmauth/src/rw)
-add_subdirectory(library/cpp/tvmauth/client/misc/retry_settings/v1)
-add_subdirectory(kikimr/yndx/grpc_services/persqueue)
-add_subdirectory(kikimr/yndx/persqueue/msgbus_server)
add_subdirectory(ydb/core/testlib/actors/ut)
add_subdirectory(ydb/core/kesus/proxy/ut)
add_subdirectory(ydb/core/kesus/tablet/ut)
diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt
index 8257d9371e..24b5927f27 100644
--- a/CMakeLists.linux.txt
+++ b/CMakeLists.linux.txt
@@ -1246,46 +1246,6 @@ add_subdirectory(ydb/core/client/metadata/ut)
add_subdirectory(ydb/core/client/minikql_compile/ut)
add_subdirectory(ydb/core/client/minikql_result_lib/ut)
add_subdirectory(ydb/core/client/server/ut)
-add_subdirectory(kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils)
-add_subdirectory(kikimr/persqueue/sdk/deprecated/cpp/v2)
-add_subdirectory(kikimr/public/sdk/cpp/client/iam)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/iam/v1)
-add_subdirectory(contrib/libs/googleapis-common-protos)
-add_subdirectory(cloud/bitbucket/common-api/yandex/cloud/api)
-add_subdirectory(cloud/bitbucket/common-api/yandex/cloud/api/tools)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/access)
-add_subdirectory(cloud/bitbucket/public-api/yandex/cloud/operation)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/access)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/oauth)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/operation)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/quota)
-add_subdirectory(cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1)
-add_subdirectory(library/cpp/http/simple)
-add_subdirectory(library/cpp/threading/atomic)
-add_subdirectory(kikimr/public/sdk/cpp/client/ydb_persqueue)
-add_subdirectory(kikimr/public/sdk/cpp/client/ydb_persqueue/codecs)
-add_subdirectory(library/cpp/streams/lzop)
-add_subdirectory(contrib/libs/minilzo)
-add_subdirectory(kikimr/yndx/api/grpc)
-add_subdirectory(kikimr/yndx/api/protos)
-add_subdirectory(kikimr/yndx/persqueue/read_batch_converter)
-add_subdirectory(library/cpp/grpc/common)
-add_subdirectory(library/cpp/tvmauth/client)
-add_subdirectory(library/cpp/openssl/crypto)
-add_subdirectory(library/cpp/openssl/big_integer)
-add_subdirectory(library/cpp/tvmauth)
-add_subdirectory(library/cpp/string_utils/secret_string)
-add_subdirectory(library/cpp/tvmauth/src/protos)
-add_subdirectory(library/cpp/tvmauth/src/rw)
-add_subdirectory(library/cpp/tvmauth/client/misc/retry_settings/v1)
-add_subdirectory(kikimr/yndx/grpc_services/persqueue)
-add_subdirectory(kikimr/yndx/persqueue/msgbus_server)
add_subdirectory(ydb/core/testlib/actors/ut)
add_subdirectory(ydb/core/kesus/proxy/ut)
add_subdirectory(ydb/core/kesus/tablet/ut)
diff --git a/cloud/README.md b/cloud/README.md
deleted file mode 100644
index 3e469f7ad8..0000000000
--- a/cloud/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Yandex Cloud (https://wiki.yandex-team.ru/cloud)
-
-This repository used for YC projects
-Another repositories - https://bb.yandex-team.ru/projects/CLOUD/
-
-Please contact devel@yandex-team.ru for any support
-
diff --git a/cloud/__init__.py b/cloud/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/cloud/__init__.py
+++ /dev/null
diff --git a/cloud/bitbucket/common-api/yandex/cloud/api/CMakeLists.txt b/cloud/bitbucket/common-api/yandex/cloud/api/CMakeLists.txt
deleted file mode 100644
index 895a0abdd9..0000000000
--- a/cloud/bitbucket/common-api/yandex/cloud/api/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yandex-cloud-api)
-set_property(TARGET yandex-cloud-api PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET yandex-cloud-api PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/common-api
-)
-target_include_directories(yandex-cloud-api PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_include_directories(yandex-cloud-api PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_link_libraries(yandex-cloud-api PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- cloud-api-tools
- contrib-libs-protobuf
-)
-target_proto_messages(yandex-cloud-api PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api/yandex/cloud/api/operation.proto
-)
-target_proto_addincls(yandex-cloud-api
- ./cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(yandex-cloud-api
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_proto_plugin(yandex-cloud-api
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/common-api/yandex/cloud/api/operation.proto b/cloud/bitbucket/common-api/yandex/cloud/api/operation.proto
deleted file mode 100644
index ad794a9cbe..0000000000
--- a/cloud/bitbucket/common-api/yandex/cloud/api/operation.proto
+++ /dev/null
@@ -1,24 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.api;
-
-import "google/protobuf/descriptor.proto";
-import "yandex/cloud/api/tools/options.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/common-api/yandex/cloud/api;api";
-option (cloud.api.tools.file).lint_skip.java_package = true;
-
-extend google.protobuf.MethodOptions { Operation operation = 87334; }
-
-// Operation is annotation for rpc that returns longrunning operation, describes
-// message types that will be returned in metadata [google.protobuf.Any], and
-// in response [google.protobuf.Any] (for successful operation).
-message Operation {
- // Optional. If present, rpc returns operation which metadata field will
- // contains message of specified type.
- string metadata = 1; // Optional.
-
- // Required. rpc returns operation, in case of success response will contains message of
- // specified field.
- string response = 2; // Required.
-}
diff --git a/cloud/bitbucket/common-api/yandex/cloud/api/tools/CMakeLists.txt b/cloud/bitbucket/common-api/yandex/cloud/api/tools/CMakeLists.txt
deleted file mode 100644
index ee192d71f6..0000000000
--- a/cloud/bitbucket/common-api/yandex/cloud/api/tools/CMakeLists.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-api-tools)
-set_property(TARGET cloud-api-tools PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-api-tools PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/common-api
-)
-target_include_directories(cloud-api-tools PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_include_directories(cloud-api-tools PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_link_libraries(cloud-api-tools PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-api-tools PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api/yandex/cloud/api/tools/options.proto
-)
-target_proto_addincls(cloud-api-tools
- ./cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-api-tools
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/common-api
-)
-target_proto_plugin(cloud-api-tools
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/common-api/yandex/cloud/api/tools/options.proto b/cloud/bitbucket/common-api/yandex/cloud/api/tools/options.proto
deleted file mode 100644
index bc0c14275c..0000000000
--- a/cloud/bitbucket/common-api/yandex/cloud/api/tools/options.proto
+++ /dev/null
@@ -1,87 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.api.tools;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/common-api/yandex/cloud/api/tools;tools";
-option (cloud.api.tools.file).lint_skip.java_package = true;
-
-// NOTE(skipor): option extention names should have package scope unique names
-// and global unique ids per Options type from range 50000-99999 reserved for internal use.
-
-extend google.protobuf.FileOptions {FileOptions file = 67321;}
-extend google.protobuf.MessageOptions {MessageOptions message = 67321;}
-extend google.protobuf.FieldOptions {FieldOptions field = 67321;}
-extend google.protobuf.EnumOptions {EnumOptions enumeration = 67321;} // enum is reserved word :(
-extend google.protobuf.EnumValueOptions {EnumValueOptions value = 67321;}
-extend google.protobuf.ServiceOptions {ServiceOptions service = 67321;}
-extend google.protobuf.MethodOptions {MethodOptions method = 67321;}
-
-message FileOptions {
- FileLintSkip lint_skip = 1;
-}
-
-message FileLintSkip {
- bool all = 1;
- bool go_package = 2;
- bool file_path = 3;
- bool java_package = 4;
-}
-
-message MessageOptions {
- CommonLintSkip lint_skip = 1;
- string openapi_name = 2;
-}
-
-message FieldOptions {
- // Used to declare which schema may have protobuf.Struct field.
- // TODO(skipor): if v1 and later apis use it move it to yandex.cloud.api package, and parse
- // at lint stage, but not at options parse.
- repeated string one_of = 1;
- FieldLintSkip lint_skip = 2;
-}
-
-message FieldLintSkip {
- bool all = 1;
- bool int_type = 2;
- bool float_type = 3;
- bool json_name = 4;
-}
-
-message EnumOptions {
- EnumLintSkip lint_skip = 2;
- string openapi_name = 3;
-}
-
-message EnumLintSkip {
- bool all = 1;
- bool value_names_case = 2;
- bool unspecified_value = 3;
-}
-
-message EnumValueOptions {
- CommonLintSkip lint_skip = 1;
-}
-
-message ServiceOptions {
- CommonLintSkip lint_skip = 1;
- bool skip_generate = 2;
-}
-
-message MethodOptions {
- MethodLintSkip lint_skip = 1;
- bool skip_generate = 2;
-}
-
-message MethodLintSkip {
- bool all = 1;
- bool contains_resource_name = 2;
- bool http_query_parameters = 3;
- bool http_path = 4;
- bool http_verb = 5;
-}
-
-message CommonLintSkip {
- bool all = 1;
-}
diff --git a/cloud/bitbucket/private-api/README.md b/cloud/bitbucket/private-api/README.md
deleted file mode 100644
index 5afc5b317e..0000000000
--- a/cloud/bitbucket/private-api/README.md
+++ /dev/null
@@ -1,52 +0,0 @@
-## Private API development guide
-
-All cloud control plane API definition based on [gRPC](https://grpc.io).
-
-### Repository setup
-#### Prerequisites
-
-- make
-- on Mac OS X, protoc (should be >= 3.5) or Homebrew (protoc will be installed using brew)
-- on Linux, protoc (should be >= 3.5) or sudo access to install it from github
-
-#### Steps
-
-Example:
-
-```
-git clone https://bb.yandex-team.ru/scm/cloud/private-api.git
-cd private-api
-
-// ...Hack-hack-hack...
-
-make lint
-```
-
-### Checking proto compilation locally
-
-Just run `make lint` inside src root.
-To build proto tools from source set 'BUILD_TOOLS' variable to 1.
-Also you may check compilation of swagger docs out of cloud API: run `make generate` for that.
-
-For Pull Request build both successful `lint` and `generate` required.
-
-
-#### Validation
-
-Normally, all the fields of all messages received from users must be validated:
-- Request messages;
-- Value objects used in request messages;
-- Value objects used in value objects and so on.
-
-Output messages does not require validation.
-
-Syntax and examples can be found in ``yandex/cloud/priv/example/v1alpha/validation_example.proto``
-
-For Java developers there is ``java`` module, ``mvn clean package`` in this directory does following:
- - builds and packages all proto-files in the repository
- - tests all validators for syntax and applicability
- - includes yandex.cloud.proto.ProtoValidator which helps validating messages in grpc interceptor.
-
-The `java/do_local_install.sh` script is useful for local development of a feature branch.
-The script will run `mvn versions:set && mvn clean install` and install built artifacts
-in the local maven repo with snapshot version. Please run `versions:revert` manually if script failed at compilation.
diff --git a/cloud/bitbucket/private-api/README.roles.md b/cloud/bitbucket/private-api/README.roles.md
deleted file mode 100644
index 8fbc10189d..0000000000
--- a/cloud/bitbucket/private-api/README.roles.md
+++ /dev/null
@@ -1,108 +0,0 @@
-# identity-role-access-matrix
-
-Этот документ описывает новый формат хранения данных о well-known сущностях IAM'а.
-
-Пермишены и сервисные роли хранятся в отдельном каталоге для сервиса, который ими управляет.
-Например, пермишен `compute.instances.start` может быть задан в файле `compute/permissions.yaml`, а роль `resource-manager.clouds.member` — в файле `resource-manager/roles.yaml`.
-Внутри своего каталога команда сервиса может организовать данные как угодно, одним файлом или несколькими, положить их в одном каталоге или раскидать по поддиректориям.
-Обязательное требование — файлы внутри подкаталогов должны называться `permissions.yaml`, `roles.yaml`, `stages.yaml`, `resources.yaml` для описания набора прав, ролей, стейджей и типов ресурсов соответсвенно.
-В каждом файле можно сослаться на сущность из любого другого файла — точно так же, как если бы сущности были описаны рядом, явно ссылаться на файл не нужно.
-
-Если эта документация противоречит тому, что на самом деле творится в файлах — значит, в файлах неправильно :)
-
-## Тулинг
-
-Для проверки того, что yaml'ы написаны верно, можно воспользоваться `yc-iam-compile-role-fixtures` из Python-пакета [yc_iam_tools](https://bb.yandex-team.ru/projects/CLOUD/repos/identity/browse/iam_tools/yc_iam_tools/). А можно и не пользоваться, такая же проверка запускается в TeamCity на каждый PR.
-
-## Роли
-
-```yaml
-roles:
- # В этом dict'е перечисляются роли: ключ — название роли, значение — dict со свойствами
-
- example.editor: # название роли
-
- # Описание роли на английском для документации.
- summary: |>
- Edit different things that are managed by ExampleService.
- Users with this role are also allowed to whisper to horses.
-
- # Видимость роли: может быть public или internal. Роли public видят пользователи, а internal роли — нет.
- # Public роль не должна включать в себя internal-пермишены, сейчас это warning при компиляции,
- # в будущем повысим до error.
- visibility: public
-
- # Минимальный тип ресурса, на который можно назначить роль.
- resourceType: resource-manager.folder
- # Эту роль можно назначить на фолдер или на клауд, но нельзя на SA или на биллинг-аккаунт.
- # Такая роль может содержать пермишены, у которых resourceType фолдер или какой-нибудь вложенный в него ресурс,
- # но не может содержать никакие другие пермишены.
-
- # Другие роли, входящие в состав этой.
- # Параметр можно не указывать, если не нужно инклюдить никакие другие роли.
- includedRoles:
- - example.viewer
- - horse.whisperer
- # Роль `example.editor` содержит все пермишены из `example.viewer` и `horse.whisperer`.
- # `includedRoles` работает транзитивно: если в определении `example.viewer` тоже инклюдятся какие-то роли,
- # то их пермишены входят и в `example.editor`.
-
- # Пермишены, входящие в роль.
- # Параметр можно не указывать, если роль не включает никаких пермишенов напрямую.
- permissions:
- - example.things.edit
- - example.things.manage
- # Есть сокращённая форма записи:
- - example.thingCollections.{create,update,delete}
- # Фигурные скобки можно использовать в любом месте записи:
- # `sample.{horses,mice,chickens}.{feed,pet}` тоже можно сказать,
- # эта запись разресолвится в 6 пермишенов.
- # (Но лучше таким не злоупотреблять.)
-
- # В итоге получается, что в роль `example.editor` входят пермишены:
- # `example.things.edit`, `example.things.manage`,
- # `example.thingCollections.create`, `example.thingCollections.update`, `example.thingCollections.delete`,
- # а также все пермишены, которые входят в роли `example.viewer` и `horse.whisperer`.
-
- # Ещё одна роль.
- example.viewer:
- # Эта роль используется в роли `example.editor` выше.
- # Но это не значит, что `example.viewer` в файле должна идти после `example.editor` —
- # можно расположить их хоть как или вообще положить в разные файлы.
- ...
-```
-
-Роли задаются аддитивно: можно создать роль "`viewer` плюс `compute.editor` плюс `iam.serviceAccounts.create`", но нельзя задать "`viewer` минус `billing.viewer`" или "все пермишены `serverless.*.*`, кроме `serverless.*.delete`". Это сделано специально, чтобы при добавлении новой роли/пермишена вся система вела себя более предсказуемо.
-
-Некоторые роли помечены как "псевдороли", у них есть поле `pseudorole: true`. Это временные сущности, они нужны только для того, чтобы из них составить общеоблачные роли типа `viewer`. Ни внешние, ни внутренние пользователи не могут видеть псевдороли в API и назначать на ресурсы. Сервисам рекомендуется заменить их на правильные сервисные роли.
-
-## Пермишены
-
-```yaml
-permissions:
-
- iam.accessBinding.delete: # имя пермишена
-
- # Описание роли на английском для документации.
- description: Delete access binding.
-
- # Стейдж. Чаще всего это GA.
- # Список стейджей лежит в `stages.yaml`.
- stage: GA
-
- # Видимость пермишена: может быть public или internal.
- # Связана с видимостью ролей: internal пермишены не должны входить в public роли.
- visibility: public
-
- # Здесь задаются условия, когда пермишен может действовать.
- allowedWhen:
-
- # Сейчас можно задать только условие на статус клауда.
- cloud:
- status:
- - BLOCKED_BY_BILLING
- - ACTIVE
- # Этим пермишеном можно воспользоваться только тогда, когда клауд
- # находится в статусе ACTIVE или BLOCKED_BY_BILLING.
- # А если клауд в другом статусе — например, BLOCKED — пермишен запрещён.
-```
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/CMakeLists.txt
deleted file mode 100644
index 4ac967704e..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/CMakeLists.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yandex-cloud-priv)
-set_property(TARGET yandex-cloud-priv PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET yandex-cloud-priv PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(yandex-cloud-priv PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(yandex-cloud-priv PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(yandex-cloud-priv PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- contrib-libs-protobuf
-)
-target_proto_messages(yandex-cloud-priv PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/sensitive.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/validation.proto
-)
-target_proto_addincls(yandex-cloud-priv
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(yandex-cloud-priv
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(yandex-cloud-priv
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/access/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/access/CMakeLists.txt
deleted file mode 100644
index 292231e480..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/access/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-priv-access)
-set_property(TARGET cloud-priv-access PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-priv-access PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-access PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-access PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(cloud-priv-access PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-priv-access PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/access/access.proto
-)
-target_proto_addincls(cloud-priv-access
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-priv-access
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(cloud-priv-access
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/access/access.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/access/access.proto
deleted file mode 100644
index 08ef897678..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/access/access.proto
+++ /dev/null
@@ -1,94 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.access;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/access;access";
-option java_outer_classname = "PA";
-
-// * `type = system, id = allUsers`: A special identifier that represents anyone.
-//
-// * `type = system, id = allAuthenticatedUsers`: A special identifier that represents anyone
-// who is authenticated.
-//
-// * `type = userAccount, id = <cloud generated id>`
-//
-// * `type = federatedUser, id = <cloud generated id>`
-//
-// * `type = serviceAccount, id = <cloud generated id>`
-//
-// * `type = group, id = <cloud generated id>`
-
-message Subject {
- string id = 1 [(required) = true, (length) = "<=50"];
- string type = 2 [(required) = true, (length) = "<=100"];
-}
-
-message AccessBinding {
- string role_id = 1 [(required) = true, (length) = "<=50"];
- Subject subject = 2 [(required) = true];
-}
-
-message ListAccessBindingsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, internal roles are not returned in a result).
- // When proxying from the public API, this field MUST be mapped to false.
- bool private_call = 4;
-}
-
-message ListAccessBindingsResponse {
- repeated AccessBinding access_bindings = 1;
- string next_page_token = 2;
-}
-
-message SetAccessBindingsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- repeated AccessBinding access_bindings = 2;
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, throwing NotFoundException on an
- // internal role). When proxying from the public API,
- // this field MUST be mapped to false.
- bool private_call = 3;
-}
-
-message SetAccessBindingsMetadata {
- string resource_id = 1;
-}
-
-message UpdateAccessBindingsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- repeated AccessBindingDelta access_binding_deltas = 2 [(size) = ">0"];
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, throwing NotFoundException on an
- // internal role). When proxying from the public API,
- // this field MUST be mapped to false.
- bool private_call = 3;
-}
-
-message UpdateAccessBindingsMetadata {
- string resource_id = 1;
-}
-
-enum AccessBindingAction {
- ACCESS_BINDING_ACTION_UNSPECIFIED = 0;
- ADD = 1;
- REMOVE = 2;
-}
-
-message AccessBindingDelta {
- AccessBindingAction action = 1 [(required) = true];
- AccessBinding access_binding = 2 [(required) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/restriction.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/restriction.proto
deleted file mode 100644
index 0f2fe12396..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/restriction.proto
+++ /dev/null
@@ -1,58 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam;iam";
-option java_outer_classname = "PRN";
-
-enum RestrictionKind {
- RESTRICTION_KIND_UNSPECIFIED = 0;
- BLOCK_PERMISSIONS = 1;
-}
-
-message Restriction {
- RestrictionKind restriction_kind = 1;
- string restriction_type_id = 2;
- google.protobuf.Timestamp added_at = 3;
- string added_by = 4;
-}
-
-message ListRestrictionsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- RestrictionKind restriction_kind = 2;
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListRestrictionsResponse {
- repeated Restriction restrictions = 1;
- string next_page_token = 2;
-}
-
-message GetRestrictionRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string restriction_type_id = 2 [(required) = true, (length) = "<=100"];
-}
-
-message AddRestrictionRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string restriction_type_id = 2 [(required) = true, (length) = "<=100"];
-}
-
-message AddRestrictionMetadata {
- string resource_id = 1;
- string restriction_type_id = 2;
-}
-
-message RemoveRestrictionRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string restriction_type_id = 2 [(required) = true, (length) = "<=100"];
-}
-
-message RemoveRestrictionMetadata {
- string resource_id = 1;
- string restriction_type_id = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/CMakeLists.txt
deleted file mode 100644
index cf42f5bfa7..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/CMakeLists.txt
+++ /dev/null
@@ -1,102 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(priv-iam-v1)
-set_property(TARGET priv-iam-v1 PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET priv-iam-v1 PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(priv-iam-v1 PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(priv-iam-v1 PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(priv-iam-v1 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-api
- cloud-api-tools
- yandex-cloud-priv
- cloud-priv-access
- iam-v1-token
- iam-v1-ts
- priv-oauth-v1
- cloud-priv-operation
- cloud-priv-quota
- priv-servicecontrol-v1
- contrib-libs-protobuf
-)
-target_proto_messages(priv-iam-v1 PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/access_binding_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/gizmo_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_cookie_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/membership_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/operation_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quota_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/root_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/subject_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token_agent.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_user_account_service.proto
-)
-target_proto_addincls(priv-iam-v1
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(priv-iam-v1
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(priv-iam-v1
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/access_binding_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/access_binding_service.proto
deleted file mode 100644
index d9b77098f0..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/access_binding_service.proto
+++ /dev/null
@@ -1,258 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/rpc/error_details.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PABS";
-
-service AccessBindingService {
- rpc ListAccessBindings (ListAccessBindingsRequest) returns (ListAccessBindingsResponse) {
- option (google.api.http) = { get: "/iam/v1/listAccessBindings" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc SetAccessBindings (SetAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/setAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "SetAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc UpdateAccessBindings (UpdateAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/updateAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- option (yandex.cloud.api.tools.method).lint_skip.http_verb = true;
- }
-
- rpc CheckLikelyPubliclyAccessible (CheckLikelyPubliclyAccessibleRequest) returns (CheckLikelyPubliclyAccessibleResponse);
-
- rpc ChangeTopLevelResource (ChangeTopLevelResourceRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "ChangeTopLevelResourceMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc DeleteByTopLevelResource (DeleteByTopLevelResourceRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteByTopLevelResourceMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-// * `type = system, id = allUsers`: A special identifier that represents anyone.
-//
-// * `type = system, id = allAuthenticatedUsers`: A special identifier that represents anyone
-// who is authenticated.
-//
-// * `type = userAccount, id = <cloud generated id>`
-//
-// * `type = federatedUser, id = <cloud generated id>`
-//
-// * `type = serviceAccount, id = <cloud generated id>`
-//
-// * `type = group, id = <cloud generated id>`
-//
-// * `type = invitee, id = <cloud generated id>`
-message Subject {
- string id = 1 [(required) = true, (length) = "<=50"];
- string type = 2 [(required) = true, (length) = "<=100"];
-}
-
-message AccessBinding {
- string role_id = 1 [(required) = true, (length) = "<=50"];
- Subject subject = 2 [(required) = true];
- string managed_by = 3 [(length) = "<=50"];
-}
-
-message ListAccessBindingsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, internal roles are not returned in a result).
- // When proxying from the public API, this field MUST be mapped to false.
- bool private_call = 3;
- int64 page_size = 4 [(value) = "<=1000"];
- string page_token = 5 [(length) = "<=100"];
-}
-
-message ListAccessBindingsResponse {
- repeated AccessBinding access_bindings = 1;
- string next_page_token = 2;
-}
-
-message SetAccessBindingsRequest {
- // Path to resource, from most to least specific according to resource hierarchy.
- // E.g. [storage.bucket, resource-manager.folder, resource-manager.cloud].
- // If specified, takes precedence over resource_id/resource_type.
- repeated servicecontrol.v1.Resource resource_path = 5;
- // Access bindings to set.
- repeated AccessBinding access_bindings = 4 [(size) = "<=1000"];
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, throwing NotFoundException on an
- // internal role). When proxying from the public API,
- // this field MUST be mapped to false.
- bool private_call = 3;
- // If set, perform operation on behalf of subject with this id.
- string on_behalf_of_subject_id = 6 [(length) = "<=50"];
-
- // Legacy resource specifiers. Use resource_path instead.
- string resource_id = 1 [(length) = "<=50"];
- string resource_type = 2 [(length) = "<=100"];
-}
-
-message SetAccessBindingsMetadata {
- string resource_id = 1;
- string resource_type = 2;
- repeated servicecontrol.v1.Resource resource_path = 3;
- string on_behalf_of_subject_id = 4;
-}
-
-message UpdateAccessBindingsRequest {
- // Path to resource, from most to least specific according to resource hierarchy
- // E.g. [storage.bucket, resource-manager.folder, resource-manager.cloud].
- // If specified, takes precedence over resource_id/resource_type.
- repeated servicecontrol.v1.Resource resource_path = 5;
- // Access bindings to add/remove.
- repeated AccessBindingDelta access_binding_deltas = 3 [(size) = "1-1000"];
- // A flag allowing the service to determine that the original action was
- // initiated from services (private API) when [private_call] = true, or from
- // the public API when [private_call] = false. [private_call] = false adds
- // additional restrictions on the execution of the action (ex. the prohibition
- // of manipulating internal roles, throwing NotFoundException on an
- // internal role). When proxying from the public API,
- // this field MUST be mapped to false.
- bool private_call = 4;
- // If set, perform operation on behalf of subject with this id.
- string on_behalf_of_subject_id = 6 [(length) = "<=50"];
-
- // Legacy resource specifiers. Use resource_path instead.
- string resource_id = 1 [(length) = "<=50"];
- string resource_type = 2 [(length) = "<=100"];
-}
-
-message UpdateAccessBindingsMetadata {
- string resource_id = 1;
- string resource_type = 2;
- repeated servicecontrol.v1.Resource resource_path = 3;
- string on_behalf_of_subject_id = 4;
-}
-
-enum AccessBindingAction {
- ACCESS_BINDING_ACTION_UNSPECIFIED = 0;
- ADD = 1;
- REMOVE = 2;
-}
-
-message AccessBindingDelta {
- AccessBindingAction action = 1 [(required) = true];
- AccessBinding access_binding = 2 [(required) = true];
-}
-
-// Access bindings operation status. Returned as an element of google.rpc.Status.details.
-// Returned only when a {set,update}AccessBindings() call is made on behalf of other subject,
-// in which case the "code" and "message" values should be returned to that subject as the GRPC result
-// of your service's {set,update}AccessBindings() method calls.
-message AccessBindingsOperationStatus {
- // End-user facing GRPC status code.
- int64 code = 1;
-
- // End-user facing message.
- string message = 2;
-
- // Private message details.
- google.rpc.DebugInfo internal = 3;
-}
-
-// Access bindings added or removed during this operation.
-message AccessBindingsOperationDelta {
- repeated AccessBindingDelta access_binding_deltas = 1;
-}
-
-// Deprecated in favor of above message.
-message BindingsOperationStatus {
- int64 code = 1 [deprecated = true];
- string message = 2 [deprecated = true];
- google.rpc.DebugInfo internal = 3 [deprecated = true];
-}
-
-message CheckLikelyPubliclyAccessibleRequest {
- // Resources to check.
- repeated servicecontrol.v1.Resource resources = 1 [(size) = "1-1000"];
-}
-
-message CheckLikelyPubliclyAccessibleResponse {
- // Resources that are likely publicly accessible;
- repeated servicecontrol.v1.Resource resources = 1;
-}
-
-message ChangeTopLevelResourceRequest {
- // Path to resource.
- repeated servicecontrol.v1.Resource resource_path = 1;
- // Top level resource to change to.
- servicecontrol.v1.Resource top_level_resource = 2;
- // Dry run flag. When this flag is true, perform preliminary check instead. In case there are
- // access bindings that would be lost in the process, returned failed operation with code FAILED_PRECONDITION
- // will contain a ChangeTopLevelResourceDryRunErrorDetails error detail describing subjects whose
- // access bindings would be lost.
- bool dry_run = 3;
-}
-
-message ChangeTopLevelResourceMetadata {
- repeated servicecontrol.v1.Resource resource_path = 1;
- servicecontrol.v1.Resource top_level_resource = 2;
- bool dry_run = 3;
-}
-
-message ChangeTopLevelResourceDryRunErrorDetails {
- // Some of the public access bindings that would be lost when changing top level resource.
- // This list might be empty in case only system bindings are present.
- // These can be displayed to public end users.
- repeated AccessBinding public_access_bindings = 1 [(size) = "<=10"];
- // Some of the system access bindings that would be lost when changing top level resource.
- // This list might be empty in case only public bindings are present.
- // These should only be displayed to staff members.
- // In case only system bindings are present, public end users should receive a vague
- // message like "cannot move access bindings", without any specifics.
- repeated AccessBinding system_access_bindings = 2 [(size) = "<=10"];
-
- message AccessBinding {
- string role_id = 1;
- string subject_id = 2;
- string resource_id = 3;
- string resource_type = 4;
- }
-}
-
-message DeleteByTopLevelResourceRequest {
- // Subject IDs whose bindings are to be removed.
- repeated string subject_ids = 1 [(size) = "1-1000"];
- // Top level resource filter of access bindings.
- servicecontrol.v1.Resource top_level_resource = 2 [(required) = true];
-}
-
-message DeleteByTopLevelResourceMetadata {
- repeated string subject_ids = 1;
- servicecontrol.v1.Resource top_level_resource = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key.proto
deleted file mode 100644
index a8ac7ac88d..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PAPIK";
-
-message ApiKey {
- string id = 1;
- string service_account_id = 2;
- google.protobuf.Timestamp created_at = 3;
- string description = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key_service.proto
deleted file mode 100644
index 6ce0ec09ff..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/api_key_service.proto
+++ /dev/null
@@ -1,104 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/api_key.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PAPIKS";
-
-service ApiKeyService {
-
- rpc List (ListApiKeysRequest) returns (ListApiKeysResponse) {
- option (google.api.http) = {get: "/iam/v1/apiKeys"};
- }
-
- rpc Get (GetApiKeyRequest) returns (ApiKey) {
- option (google.api.http) = {get: "/iam/v1/apiKeys/{api_key_id}"};
- }
-
- rpc Create (CreateApiKeyRequest) returns (CreateApiKeyResponse) {
- option (google.api.http) = {post: "/iam/v1/apiKeys" body: "*"};
- }
-
- rpc Update (UpdateApiKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {patch: "/iam/v1/apiKeys/{api_key_id}" body: "*"};
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateApiKeyMetadata"
- response: "ApiKey"
- };
- }
-
- rpc Delete (DeleteApiKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {delete: "/iam/v1/apiKeys/{api_key_id}"};
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteApiKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc ListOperations (ListApiKeyOperationsRequest) returns (ListApiKeyOperationsResponse) {
- option (google.api.http) = {get: "/iam/v1/apiKeys/{api_key_id}/operations"};
- }
-}
-
-message GetApiKeyRequest {
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListApiKeysRequest {
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListApiKeysResponse {
- repeated ApiKey api_keys = 1;
- string next_page_token = 2;
-}
-
-message CreateApiKeyRequest {
- string api_key_id = 3 [(required) = false, (length) = "<=50"];
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
- string description = 2 [(length) = "<=256"];
-}
-
-message CreateApiKeyResponse {
- ApiKey api_key = 1;
- string secret = 2 [(sensitive) = true];
-}
-
-message UpdateApiKeyRequest {
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateApiKeyMetadata {
- string api_key_id = 1;
-}
-
-message DeleteApiKeyRequest {
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteApiKeyMetadata {
- string api_key_id = 1;
-}
-
-message ListApiKeyOperationsRequest {
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListApiKeyOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto
deleted file mode 100644
index c2a3dafac3..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.awscompatibility;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility;awscompatibility";
-option java_outer_classname = "PAK";
-
-message AccessKey {
- string id = 1;
- string service_account_id = 2;
- google.protobuf.Timestamp created_at = 3;
- string description = 4;
- string key_id = 5;
- // Base64-encoded (no padding) sha256 hash of binary secret key
- string fingerprint = 6;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key_service.proto
deleted file mode 100644
index 01070526f6..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility/access_key_service.proto
+++ /dev/null
@@ -1,117 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.awscompatibility;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/awscompatibility;awscompatibility";
-option java_outer_classname = "PAKS";
-
-service AccessKeyService {
-
- rpc List (ListAccessKeysRequest) returns (ListAccessKeysResponse) {
- option (google.api.http) = {get: "/iam/v1/accessKeys"};
- }
-
- rpc ListByFingerprint (ListAccessKeysByFingerprintRequest) returns (ListAccessKeysByFingerprintResponse);
-
- rpc Get (GetAccessKeyRequest) returns (AccessKey) {
- option (google.api.http) = {get: "/iam/v1/accessKeys/{access_key_id}"};
- }
-
- rpc Create (CreateAccessKeyRequest) returns (CreateAccessKeyResponse) {
- option (google.api.http) = {post: "/iam/v1/accessKeys" body: "*"};
- }
-
- rpc Update (UpdateAccessKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {patch: "/iam/v1/accessKeys/{access_key_id}" body: "*"};
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateAccessKeyMetadata"
- response: "AccessKey"
- };
- }
-
- rpc Delete (DeleteAccessKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {delete: "/iam/v1/accessKeys/{access_key_id}"};
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteAccessKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc ListOperations (ListAccessKeyOperationsRequest) returns (ListAccessKeyOperationsResponse) {
- option (google.api.http) = {get: "/iam/v1/accessKeys/{access_key_id}/operations"};
- }
-}
-
-message GetAccessKeyRequest {
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListAccessKeysRequest {
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessKeysResponse {
- repeated AccessKey access_keys = 1;
- string next_page_token = 2;
-}
-
-message ListAccessKeysByFingerprintRequest {
- string fingerprint = 1 [(length) = "43"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessKeysByFingerprintResponse {
- repeated AccessKey access_keys = 1;
- string next_page_token = 2;
-}
-
-message CreateAccessKeyRequest {
- string access_key_id = 3 [(required) = false, (length) = "<=50"];
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
- string description = 2 [(length) = "<=256"];
-}
-
-message CreateAccessKeyResponse {
- AccessKey access_key = 1;
- string secret = 2 [(sensitive) = true];
-}
-
-message UpdateAccessKeyRequest {
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateAccessKeyMetadata {
- string access_key_id = 1;
-}
-
-message DeleteAccessKeyRequest {
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteAccessKeyMetadata {
- string access_key_id = 1;
-}
-
-message ListAccessKeyOperationsRequest {
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessKeyOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/access_binding_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/access_binding_service.proto
deleted file mode 100644
index 73979fc24e..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/access_binding_service.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.backoffice;
-
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice;iam_backoffice";
-option java_outer_classname = "PBOABS";
-
-service AccessBindingService {
- rpc ListBySubject (ListSubjectAccessBindingsRequest) returns (ListSubjectAccessBindingsResponse);
-}
-
-message ListSubjectAccessBindingsRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "<=1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListSubjectAccessBindingsResponse {
- repeated SubjectAccessBinding access_bindings = 1;
- string next_page_token = 2;
-}
-
-message SubjectAccessBinding {
- string role_id = 1;
- servicecontrol.v1.Resource resource = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/permission_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/permission_service.proto
deleted file mode 100644
index e808df5fe1..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice/permission_service.proto
+++ /dev/null
@@ -1,49 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.backoffice;
-
-import "google/rpc/status.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/servicecontrol/v1/access_service.proto";
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/backoffice;iam_backoffice";
-option java_outer_classname = "PBOPS";
-
-service PermissionService {
- rpc Check (CheckPermissionRequest) returns (CheckPermissionResponse);
-}
-
-message CheckPermissionRequest {
- // Permission to check.
- string permission = 1 [(required) = true, (length) = "<=50"];
- // Resource path from most specific resource to least.
- // Same semantics as AccessService.AuthorizeRequest#resource_path.
- repeated servicecontrol.v1.Resource resource_path = 2 [(size) = "<=50"];
- // Subject to check permission for. Use caller subject if not specified.
- servicecontrol.v1.Subject subject = 3;
-}
-
-message CheckPermissionResponse {
- // Permission info for resources in specified resource path, from most specific to least.
- // Might contain more entries than specified resource path.
- repeated ResourcePermissionInfo resource_permissions = 1;
-}
-
-message ResourcePermissionInfo {
- // Resource against which the permission is being checked.
- servicecontrol.v1.Resource resource = 1;
-
- // Access bindings on this resource that grant the permission.
- repeated access.AccessBinding access_bindings = 2;
-
- // AccessService response trying to authorize the permission against this resource.
- oneof authorize_response {
- option (exactly_one) = true;
- // If authorization succeeded.
- servicecontrol.v1.AuthorizeResponse response = 3;
- // If authorization failed.
- google.rpc.Status status = 4;
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/compute/os_login_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/compute/os_login_service.proto
deleted file mode 100644
index 53f207c58f..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/compute/os_login_service.proto
+++ /dev/null
@@ -1,86 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.compute;
-
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/compute;iam_compute";
-option java_outer_classname = "PCOLS";
-
-service OsLoginService {
- // DEPRECATED.
- rpc GetUserInfo (GetUserInfoRequest) returns (GetUserInfoResponse);
-
- rpc Get (GetOsLoginRequest) returns (GetOsLoginResponse);
- rpc List (ListOsLoginsRequest) returns (ListOsLoginsResponse);
-}
-
-message GetUserInfoRequest {
- // ID of the cloud to search for os login in.
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- // A resource path for compute instance. Same one to be passed to AccessService#authorize().
- // Must consist of [compute.instance, resource-manager.folder] in that order.
- // cloud_id is automatically appended from request.
- repeated servicecontrol.v1.Resource instance_resource_path = 2 [(size) = "2"];
- oneof subject {
- option (exactly_one) = true;
- string subject_id = 3 [(length) = "<=50"];
- string os_login = 4 [(length) = "<=32"];
- int64 os_uid = 5 [(value) = ">=0"];
- }
-}
-
-message GetUserInfoResponse {
- string subject_id = 1;
- string os_login = 2;
- int64 os_uid = 3;
- bool is_admin = 4;
- repeated SshKeyInfo ssh_keys = 5;
-}
-
-message GetOsLoginRequest {
- // ID of the cloud to search for os login in.
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- oneof subject {
- option (exactly_one) = true;
- string subject_id = 3 [(length) = "<=50"];
- string os_login = 4 [(length) = "<=32"];
- int64 os_uid = 5 [(value) = ">=0"];
- }
-}
-
-message GetOsLoginResponse {
- OsLoginInfo os_login_info = 1;
-}
-
-message ListOsLoginsRequest {
- // ID of the cloud to search for os login in.
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "<=2048"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListOsLoginsResponse {
- repeated OsLoginInfo os_login_infos = 1;
- string next_page_token = 2;
-}
-
-message OsLoginInfo {
- string subject_id = 1;
- SubjectType subject_type = 2;
- string os_login = 3;
- int64 os_uid = 4;
- repeated SshKeyInfo ssh_keys = 5;
-}
-
-enum SubjectType {
- SUBJECT_TYPE_UNSPECIFIED = 0;
- USER_ACCOUNT = 1;
- SERVICE_ACCOUNT = 2;
-}
-
-message SshKeyInfo {
- string id = 1;
- string data = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/access_binding_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/access_binding_service.proto
deleted file mode 100644
index b44832b698..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/access_binding_service.proto
+++ /dev/null
@@ -1,60 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/oauth/claims.proto";
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCABS";
-
-// Console-specific AccessBindingService.
-// Analogous to the regular ABS from private API, but to be used by service facades when serving
-// console-specific access bindings calls.
-// Usage scenario:
-// [end user] --access-bindings-UI--> [console] --> [console folder service] --> [console ABS]
-
-// Important thing to note here is that access bindings listing response is leaking information.
-// By providing "inherited_from" field, access bindings from resources other than specified one
-// are inadvertently disclosed. Those are the access bindings the user might not have had permissions
-// to read.
-// It was decided however to greenlight this approach since it was considered more "harmful" to
-// not let the end user see that other subjects might have access to his generally private resources,
-// rather than trying to maximize security on a method level.
-
-service AccessBindingService {
- rpc ListAccessBindings (ListAccessBindingsRequest) returns (ListAccessBindingsResponse) {
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-}
-
-message ListAccessBindingsRequest {
- // Path to resource, from most to least specific according to resource hierarchy.
- // Access bindings from all resources but first are considered "inherited" by the first resource.
- repeated servicecontrol.v1.Resource resource_path = 1 [(size) = "1-10"];
- int64 page_size = 2 [(value) = "<=1000"];
- string page_token = 3 [(length) = "<=100"];
- // Access bindings listing filter.
- // Filter specs: https://wiki.yandex-team.ru/users/zdazzy/cloud/iam/service/console/objectfilter/
- string filter = 4 [(length) = "<=1000"];
- bool get_inherited_bindings = 5;
-}
-
-message ListAccessBindingsResponse {
- repeated SubjectWithBindings subjects_with_bindings = 1;
- string next_page_token = 3;
-}
-
-message SubjectWithBindings {
- yandex.cloud.priv.oauth.SubjectClaims subject_claims = 1;
- repeated AccessBinding access_bindings = 2;
- repeated AccessBinding inherited_access_bindings = 3;
-}
-
-message AccessBinding {
- string role_id = 1;
- // If present, specifies a resource this access binding is inherited from.
- servicecontrol.v1.Resource inherited_from = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/key_service.proto
deleted file mode 100644
index 1881d2ce28..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/key_service.proto
+++ /dev/null
@@ -1,28 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/priv/iam/v1/key.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCKS";
-
-service KeyService {
-
- rpc Create (CreateKeyRequest) returns (CreateKeyResponse);
-
-}
-
-message CreateKeyRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
- string description = 4 [(length) = "<=256"];
- Key.Algorithm key_algorithm = 5;
-}
-
-message CreateKeyResponse {
- Key key = 1;
- string private_key = 2 [(sensitive) = true];
- string key_json = 3 [(sensitive) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/membership_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/membership_service.proto
deleted file mode 100644
index cf88c8c06c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/membership_service.proto
+++ /dev/null
@@ -1,39 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/priv/oauth/claims.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCMS";
-
-// Console-specific MembershipService.
-
-service MembershipService {
- rpc ListMembers (ListMembersRequest) returns (ListMembersResponse);
-}
-
-message ListMembersRequest {
- oneof list_context {
- option (exactly_one) = true;
- string organization_id = 1 [(length) = "<=50"];
- string cloud_id = 2 [(length) = "<=50"];
- }
-
- // Subjects listing filter.
- // Filter specs: https://wiki.yandex-team.ru/users/zdazzy/cloud/iam/service/console/objectfilter/
- string filter = 3 [(length) = "<=1000"];
-
- int64 page_size = 4 [(value) = "<=1000"];
- string page_token = 5 [(length) = "<=100"];
-}
-
-message ListMembersResponse {
- repeated SubjectWithClaims members = 1;
- string next_page_token = 2;
-}
-
-message SubjectWithClaims {
- yandex.cloud.priv.oauth.SubjectClaims subject_claims = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/os_login_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/os_login_service.proto
deleted file mode 100644
index 5507334b15..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/os_login_service.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCOLS";
-
-service OsLoginService {
- // Get OS Login info for current subject.
- rpc Get (GetOsLoginRequest) returns (GetOsLoginResponse);
-}
-
-message GetOsLoginRequest {
- // ID of the cloud to search for OS Login if called by a non-service account.
- string cloud_id = 1 [(length) = "<=50"];
-}
-
-message GetOsLoginResponse {
- string subject_id = 1;
- string os_login = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/role_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/role_service.proto
deleted file mode 100644
index 3e83afd99a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/role_service.proto
+++ /dev/null
@@ -1,46 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCRS";
-
-// Console-specific RoleService.
-
-service RoleService {
- rpc List (ListRolesRequest) returns (ListRolesResponse);
- rpc ListCategories (ListCategoriesRequest) returns (ListCategoriesResponse);
-}
-
-message ListRolesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListRolesResponse {
- repeated Role roles = 1;
- string next_page_token = 2;
-}
-
-message Role {
- string id = 1;
- string description = 2;
- repeated string category_ids = 3;
-}
-
-message ListCategoriesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListCategoriesResponse {
- repeated Category categories = 1;
- string next_page_token = 2;
-}
-
-message Category {
- string id = 1;
- string name = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/service_account_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/service_account_service.proto
deleted file mode 100644
index ea4a43bfdd..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/service_account_service.proto
+++ /dev/null
@@ -1,58 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/console/v1/access_binding.proto";
-import "yandex/cloud/priv/iam/v1/service_account.proto";
-import "yandex/cloud/priv/iam/v1/awscompatibility/access_key.proto";
-import "yandex/cloud/priv/iam/v1/key.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCSAS";
-
-// Console-specific ServiceAccountService.
-
-service ServiceAccountService {
- rpc List (ListServiceAccountsRequest) returns (ListServiceAccountsResponse);
-
- rpc Create (CreateServiceAccountRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "CreateServiceAccountMetadata"
- response: "ServiceAccount"
- };
- }
-
- rpc ListAccessBindings (yandex.cloud.priv.console.v1.ListAccessBindingsRequest) returns (yandex.cloud.priv.console.v1.ListAccessBindingsResponse);
-}
-
-message ListServiceAccountsRequest {
- string folder_id = 1 [(length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ServiceAccountWithKeys {
- ServiceAccount service_account = 1;
- repeated yandex.cloud.priv.iam.v1.awscompatibility.AccessKey access_keys = 2;
- repeated yandex.cloud.priv.iam.v1.Key authorized_keys = 3;
-}
-
-message ListServiceAccountsResponse {
- repeated ServiceAccountWithKeys service_accounts = 1;
- string next_page_token = 2;
-}
-
-message CreateServiceAccountRequest {
- string folder_id = 1 [(required) = true, (length) = "<=50"];
- string name = 2 [(required) = true, (pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 3 [(length) = "<=256"];
- map<string, string> labels = 4 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
- // Folder ID on which to place access bindings with roles specified below.
- // Can be different from folder_id, in which SA itself resides.
- string roles_folder_id = 5 [(required) = true, (length) = "<=50"];
- // Roles to grant to this SA.
- repeated string roles = 6 [(size) = "1-10", (length) = "<=50"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/user_account_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/user_account_service.proto
deleted file mode 100644
index 454dd2791c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console/user_account_service.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.console;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/console;iam_console";
-option java_outer_classname = "PCUAS";
-
-service UserAccountService {
-
- // Deletes all access keys for the current authenticated user.
- rpc RevokeAllAccessKeys (RevokeAllAccessKeysRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "RevokeAllAccessKeysMetadata"
- response: "google.protobuf.Empty"
- };
-
- }
-
-}
-
-message RevokeAllAccessKeysRequest {
-}
-
-message RevokeAllAccessKeysMetadata {
- string subject_id = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/gizmo_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/gizmo_service.proto
deleted file mode 100644
index 2aa8f0ca00..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/gizmo_service.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PGS";
-
-service GizmoService {
- rpc ListAccessBindings (ListGizmoAccessBindingsRequest) returns (access.ListAccessBindingsResponse);
-
- rpc UpdateAccessBindings (UpdateGizmoAccessBindingsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateGizmoAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message ListGizmoAccessBindingsRequest {
- int64 page_size = 1 [(value) = "<=1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message UpdateGizmoAccessBindingsRequest {
- repeated access.AccessBindingDelta access_binding_deltas = 1 [(size) = ">0"];
-}
-
-message UpdateGizmoAccessBindingsMetadata {
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_cookie_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_cookie_service.proto
deleted file mode 100644
index a1060c3753..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_cookie_service.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto";
-import "yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto";
-import "yandex/cloud/priv/oauth/v1/oauth_request.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PICS";
-
-service IamCookieService {
- rpc Create (CreateIamCookieRequest) returns (CreateIamCookieResponse);
- rpc CreateForUserAccount (CreateIamCookieForSubjectRequest) returns (CreateIamCookieResponse) {
- option deprecated = true;
- }
- rpc CreateForSubject (CreateIamCookieForSubjectRequest) returns (CreateIamCookieResponse);
- rpc Revoke (RevokeCookieRequest) returns (RevokeCookieResponse);
-}
-
-message CreateIamCookieForSubjectRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
- yandex.cloud.priv.oauth.v1.OAuthRequest oauth_request = 2;
- // additional parameters for cross-domain authentication
- string session_id = 3;
-}
-
-message CreateIamCookieRequest {
- oneof identity {
- option (exactly_one) = true;
- string user_account_id = 1 [deprecated = true];
- YandexPassportCookies yandex_passport_cookies = 2;
- }
- yandex.cloud.priv.oauth.v1.OAuthRequest oauth_request = 10;
- // additional parameters for cross-domain authentication
- string session_id = 11;
-}
-
-message RevokeCookieRequest {
- string iam_cookie = 1 [(required) = true, (sensitive) = true, (sensitive_type) = SENSITIVE_IAM_COOKIE];
- YandexPassportCookies yandex_passport_cookies = 2;
-}
-
-message RevokeCookieResponse {
- yandex.cloud.priv.iam.v1.ts.Subject subject = 1;
-}
-
-message CreateIamCookieResponse {
- string iam_cookie = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_COOKIE];
- google.protobuf.Timestamp issued_at = 4;
- google.protobuf.Timestamp expires_at = 2;
- yandex.cloud.priv.iam.v1.ts.Subject subject = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.proto
deleted file mode 100644
index 5f7da38c5a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.proto
+++ /dev/null
@@ -1,69 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto";
-import "yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto";
-import "yandex/cloud/priv/oauth/v1/oauth_request.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PITS";
-
-service IamTokenService {
-
- // method doesn't require auth
- rpc Create (CreateIamTokenRequest) returns (CreateIamTokenResponse) {
- option (google.api.http) = { post: "/iam/v1/tokens" body: "*" };
- }
-
- // create iam token for service account
- rpc CreateForServiceAccount (CreateIamTokenForServiceAccountRequest) returns (CreateIamTokenResponse);
-
- // create iam token for compute instance
- rpc CreateForComputeInstance (CreateIamTokenForComputeInstanceRequest) returns (CreateIamTokenResponse);
-
- // create iam token for oauth request
- rpc CreateForUserAccount (CreateIamTokenForSubjectRequest) returns (CreateIamTokenResponse) {
- option deprecated = true;
- }
-
- // create iam token for oauth request
- rpc CreateForSubject (CreateIamTokenForSubjectRequest) returns (CreateIamTokenResponse);
-}
-
-message CreateIamTokenRequest {
- oneof identity {
- option (exactly_one) = true;
- string yandex_passport_oauth_token = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_YANDEX_PASSPORT_OAUTH_TOKEN, (length) = "<=4000"];
- string jwt = 2 [(sensitive) = true, (length) = "<=8000"];
- string iam_cookie = 3 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_COOKIE];
- YandexPassportCookies yandex_passport_cookies = 4;
- }
-}
-
-message CreateIamTokenForSubjectRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
- yandex.cloud.priv.oauth.v1.OAuthRequest oauth_request = 2;
- // additional parameters for cross-domain authentication
- string session_id = 3;
-}
-
-message CreateIamTokenForServiceAccountRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message CreateIamTokenForComputeInstanceRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
- string instance_id = 2 [(required) = true, (length) = "<=50"];
-}
-
-message CreateIamTokenResponse {
- string iam_token = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
- google.protobuf.Timestamp issued_at = 4;
- google.protobuf.Timestamp expires_at = 2;
- yandex.cloud.priv.iam.v1.ts.Subject subject = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key.proto
deleted file mode 100644
index dcbf374296..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PK";
-
-message Key {
- enum Algorithm {
- ALGORITHM_UNSPECIFIED = 0;
- RSA_2048 = 1;
- RSA_4096 = 2;
- }
-
- string id = 1;
-
- oneof subject {
- string user_account_id = 2;
- string service_account_id = 3;
- }
-
- google.protobuf.Timestamp created_at = 4;
- string description = 5;
-
- Algorithm key_algorithm = 6;
- string public_key = 7;
-
- // Base64-encoded (no padding) sha256 hash of DER-encoded RSA public key in SubjectPublicKeyInfo format.
- string fingerprint = 8;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key_service.proto
deleted file mode 100644
index 41e2ea4eb9..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/key_service.proto
+++ /dev/null
@@ -1,140 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/iam/v1/key.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PKS";
-
-service KeyService {
-
- rpc Get (GetKeyRequest) returns (Key) {
- option (google.api.http) = { get: "/iam/v1/keys/{key_id}" };
- }
-
- rpc List (ListKeysRequest) returns (ListKeysResponse) {
- option (google.api.http) = { get: "/iam/v1/keys" };
- }
-
- rpc ListByFingerprint (ListKeysByFingerprintRequest) returns (ListKeysByFingerprintResponse);
-
- rpc Create (CreateKeyRequest) returns (CreateKeyResponse) {
- option (google.api.http) = { post: "/iam/v1/keys" body: "*" };
- }
-
- rpc Update (UpdateKeyRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/keys/{key_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateKeyMetadata"
- response: "Key"
- };
- }
-
- rpc Delete (DeleteKeyRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/keys/{key_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc ListOperations (ListKeyOperationsRequest) returns (ListKeyOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/keys/{key_id}/operations" };
- }
-}
-
-message GetKeyRequest {
- string key_id = 1 [(length) = "<=50"];
- KeyFormat format = 2;
-}
-
-message ListKeysRequest {
- KeyFormat format = 1;
- // one of service_account_id, user_account_id or federated_user_id.
- // cannot be made oneof{} in spec due to public API incompatibility issues.
- // https://bb.yandex-team.ru/projects/CLOUD/repos/cloud-go/pull-requests/7995/overview?commentId=1093148
- // use calling subject if not specified
- string service_account_id = 2 [(length) = "<=50"];
- string user_account_id = 5 [(length) = "<=50"];
- string federated_user_id = 6 [(length) = "<=50"];
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListKeysResponse {
- repeated Key keys = 1;
- string next_page_token = 2;
-}
-
-message ListKeysByFingerprintRequest {
- string fingerprint = 1 [(length) = "43"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListKeysByFingerprintResponse {
- repeated Key keys = 1;
- string next_page_token = 2;
-}
-
-message CreateKeyRequest {
- string key_id = 5 [(length) = "<=50"];
- // one of service_account_id, user_account_id or federated_user_id.
- // cannot be made oneof{} in spec due to public API incompatibility issues.
- // https://bb.yandex-team.ru/projects/CLOUD/repos/cloud-go/pull-requests/7995/overview?commentId=1093148
- // use calling subject if not specified
- string service_account_id = 1 [(length) = "<=50"];
- string user_account_id = 7 [(length) = "<=50"];
- string federated_user_id = 8 [(length) = "<=50"];
- string description = 2 [(length) = "<=256"];
- KeyFormat format = 3;
- Key.Algorithm key_algorithm = 4;
- string public_key = 6 [(length) = "<=15000"]; // create key with custom public part. no private key in response.
-}
-
-message CreateKeyResponse {
- Key key = 1;
- string private_key = 2 [(sensitive) = true];
-}
-
-message UpdateKeyRequest {
- string key_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateKeyMetadata {
- string key_id = 1;
-}
-
-message DeleteKeyRequest {
- string key_id = 1 [(length) = "<=50"];
-}
-
-message DeleteKeyMetadata {
- string key_id = 1;
-}
-
-message ListKeyOperationsRequest {
- string key_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListKeyOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
-
-enum KeyFormat {
- option (cloud.api.tools.enumeration).lint_skip.unspecified_value = true;
- PEM_FILE = 0;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/membership_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/membership_service.proto
deleted file mode 100644
index 6dec6e5ef7..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/membership_service.proto
+++ /dev/null
@@ -1,50 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PMS";
-
-service MembershipService {
- rpc ListResourceMembers (ListResourceMembersRequest) returns (ListResourceMembersResponse);
-
- // List member resources for specified resource type. Response resources are sorted by resource_id.
- rpc ListMemberResources (ListMemberResourcesRequest) returns (ListMemberResourcesResponse);
-}
-
-// List all members of a resource of type "resource_type" with id "resource_id".
-message ListResourceMembersRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListResourceMembersResponse {
- repeated ResourceMember resource_members = 1;
- string next_page_token = 2;
-}
-
-message ResourceMember {
- string subject_id = 1;
-}
-
-// List all resources of type "resource_type" in which "subject_id" is a member.
-message ListMemberResourcesRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListMemberResourcesResponse {
- repeated MemberResource member_resources = 1;
- string next_page_token = 2;
-}
-
-message MemberResource {
- string resource_id = 1;
- string resource_type = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/console/totp_profile_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/console/totp_profile_service.proto
deleted file mode 100644
index 8d34563bc3..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/console/totp_profile_service.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa.console;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/iam/v1/mfa/totp_profile.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/console;mfa_console";
-option java_outer_classname = "PCTPS";
-
-// A set of methods for managing hardware and software time-based one time passwords (TOTP).
-// The user credentials should be passed in the authorization header.
-service TotpProfileService {
-
- // Verifies user-supplied TOTP value. See https://tools.ietf.org/html/rfc6238#section-5.2 for the reference.
- rpc Verify (VerifyTotpRequest) returns (VerifyTotpResponse) {}
-}
-
-message VerifyTotpRequest {
- int64 code = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_REMOVE, (value) = ">0"];
- string totp_profile_id = 2 [(required) = true, (length) = "<=50"];
-}
-
-message VerifyTotpResponse {
- VerificationResult result = 1;
- // HTTP-header Set-Cookie for End-User with required per-service cookies, e.g. yc_session.
- // See also yandex/cloud/priv/oauth/v1/session_service.proto specification.
- repeated string set_cookie_header = 2 [(sensitive) = true];
- google.protobuf.Timestamp retry_at = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile.proto
deleted file mode 100644
index 83734ff164..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile.proto
+++ /dev/null
@@ -1,43 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa.hardware;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/iam/v1/mfa/totp_profile.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware;mfa_hardware";
-option java_outer_classname = "PHTP";
-
-message TotpProfile {
- // TOTP profile ID.
- string id = 1;
- // The organization in which the hardware token is registered
- string organization_id = 2;
- // The user that this token binds to.
- string subject_id = 3;
- google.protobuf.Timestamp created_at = 4;
- // True, if the user has ever been successfully verified.
- bool active = 5;
- // Descriptions of profile
- string description = 6;
- // Labels of profile
- map<string, string> labels = 7;
- // The algorithm used to calculate the hash.
- HashAlgorithm algorithm = 8;
- // The length of the TOTP code (6-8).
- int64 digits = 9;
- // The period that a TOTP code will be valid for.
- google.protobuf.Duration time_step = 10 [(value) = "10s-10m"];
- // Serial number hardware token
- string serial_no = 11;
- // Manufacturer hardware token
- string manufacturer = 12;
- // Model hardware token
- string model = 13;
- // Start time for the hardware token. Default is Unix Epoch.
- google.protobuf.Timestamp start_time = 14;
- // Token time lags
- google.protobuf.Duration time_drift = 15;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile_service.proto
deleted file mode 100644
index 3ec2456c84..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile_service.proto
+++ /dev/null
@@ -1,243 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa.hardware;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/field_mask.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/iam/v1/mfa/hardware/totp_profile.proto";
-import "yandex/cloud/priv/iam/v1/mfa/totp_profile.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/hardware;mfa_hardware";
-option java_outer_classname = "PHTPS";
-
-// A set of methods for managing hardware time-based one time passwords (TOTP).
-// The user credentials should be passed in the authorization header.
-service TotpProfileService {
-
- // Creates hardware TOTP profile
- rpc Create (CreateTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "CreateTotpProfileMetadata"
- response: "TotpProfile"
- };
- }
-
- // Bulk creates hardware TOTP profiles using file .csv
- rpc ImportFromFile (ImportTotpProfilesRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "ImportTotpProfilesMetadata"
- response: "ImportTotpProfilesResponse"
- };
- }
-
- // Retrieves hardware TOTP profile by Id
- rpc Get (GetTotpProfileRequest) returns (TotpProfile) {}
-
- // Retrieves the list hardware TOTP profiles by organization_id and filter
- rpc List (ListTotpProfilesRequest) returns (ListTotpProfilesResponse) {}
-
- // Updates hardware TOTP profile
- rpc Update (UpdateTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateTotpProfileMetadata"
- response: "TotpProfile"
- };
- }
-
- // Updates hardware TOTP profile by id
- rpc Delete (DeleteTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteTotpProfileMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Retrieves the list of Operations for the specified hardware TOTP profile.
- rpc ListOperations (ListTotpProfileOperationsRequest) returns (ListTotpProfileOperationsResponse) {}
-
- // Binds the token to the user for HelpDesk
- rpc Assign (AssignTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "AssignTotpProfileMetadata"
- response: "TotpProfile"
- };
- }
-
- // Unbind the token from the user for HelpDesk
- rpc Unassign (UnassignTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UnassignTotpProfileMetadata"
- response: "TotpProfile"
- };
- }
-
- // Synchronizes clocks for hardware tokens. Doesn't activate profile.
- rpc SyncTimeDrift (SyncTimeDriftRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "SyncTimeDriftMetadata"
- response: "TotpProfile"
- };
- }
-
- // Verifies user-supplied TOTP value. See https://tools.ietf.org/html/rfc6238#section-5.2 for the reference.
- rpc Verify (VerifyTotpRequest) returns (VerifyTotpResponse) {}
-}
-
-message CreateTotpProfileRequest {
-
- // The organization in which the hardware token is registered
- string organization_id = 1 [(required) = true, (length) = "<=50"];
-
- // An optional subject_id to auto-assign
- string subject_id = 2 [(length) = "<=50"];
-
- // Optional description
- string description = 3 [(length) = "<=256"];
-
- // Optional labels
- map<string, string> labels = 4 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-
- // Algorithm used to calculate the hash (SHA1, SHA256 or SHA512). Default is SHA1.
- HashAlgorithm algorithm = 5;
-
- // Length of the TOTP code (6, 7 or 8). Default is 6.
- int64 digits = 6 [(value) = "0,6,7,8"];
-
- // The period that a TOTP code will be valid for.
- google.protobuf.Duration time_step = 7 [(value) = "10s-10m"];
-
- // secret encoded in HEX
- string secret = 8 [(required) = true, (sensitive) = true, (length) = "<=262144"];
-
- // Serial number hardware token
- string serial_no = 9 [(required) = true, (pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
-
- // Manufacturer hardware token
- string manufacturer = 10 [(length) = "<=256"];
-
- // Model hardware token
- string model = 11 [(length) = "<=256"];
-
- // Start time for the hardware token. Default is Unix Epoch.
- google.protobuf.Timestamp start_time = 12;
-
- // Token time lags
- google.protobuf.Duration time_drift = 13;
-}
-
-message CreateTotpProfileMetadata {
- string totp_profile_id = 1;
-}
-
-message ImportTotpProfilesRequest {
- // Data from file csv, secret encoded in HEX
- string data = 1 [(required) = true, (length) = "<=262144", (sensitive) = true];
- HashAlgorithm algorithm = 2;
- int64 digits = 3 [(value) = "0,6,7,8"];
- google.protobuf.Duration time_step = 4 [(value) = "10s-10m"];
- string description = 5 [(length) = "<=256"];
- map<string, string> labels = 6 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
- string manufacturer = 7 [(length) = "<=256"];
- string model = 8 [(length) = "<=256"];
-}
-
-message ImportTotpProfilesMetadata {
-}
-
-message ImportTotpProfilesResponse {
- repeated hardware.TotpProfile totp_profiles = 1;
-}
-
-message GetTotpProfileRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListTotpProfilesRequest {
- string organization_id = 1 [(required) = true, (length) = "<=50"];
- string filter = 2 [(length) = "<=1000"];
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListTotpProfilesResponse {
- repeated hardware.TotpProfile totp_profiles = 1;
- string next_page_token = 2;
-}
-
-message UpdateTotpProfileRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
- // Mask specifying which totp profile fields will be updated.
- google.protobuf.FieldMask update_mask = 2;
- string description = 3 [(length) = "<=256"];
- map<string, string> labels = 4 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-}
-
-message UpdateTotpProfileMetadata {
- string totp_profile_id = 1;
-}
-
-message DeleteTotpProfileRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteTotpProfileMetadata {
- string totp_profile_id = 1;
-}
-
-message ListTotpProfileOperationsRequest {
- oneof from {
- option (exactly_one) = true;
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
- string organization_id = 2 [(required) = true, (length) = "<=50"];
- }
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListTotpProfileOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
-
-message AssignTotpProfileRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
- string subject_id = 2 [(required) = true, (length) = "<=50"];
-}
-
-message AssignTotpProfileMetadata {
- string totp_profile_id = 1;
-}
-
-message UnassignTotpProfileRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message UnassignTotpProfileMetadata {
- string totp_profile_id = 1;
-}
-
-message SyncTimeDriftRequest {
- string totp_profile_id = 1 [(required) = true, (length) = "<=50"];
- repeated int64 code = 2 [(sensitive) = true, (sensitive_type) = SENSITIVE_REMOVE, (value) = ">0", (size) = ">1"];
- // Maximum number of steps time_step by which the time of the server and the token can differ. Default value is 25
- int64 window_size = 3 [(value) = "1-100"];
-}
-
-message SyncTimeDriftMetadata {
- string totp_profile_id = 1;
-}
-
-message VerifyTotpRequest {
- int64 code = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_REMOVE, (value) = ">0"];
- string totp_profile_id = 2 [(required) = true, (length) = "<=50"];
-}
-
-message VerifyTotpResponse {
- VerificationResult result = 1;
- google.protobuf.Timestamp retry_at = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/operation_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/operation_service.proto
deleted file mode 100644
index 56b9c5b4c1..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/operation_service.proto
+++ /dev/null
@@ -1,17 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa;
-
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa;mfa";
-option java_outer_classname = "POS";
-
-service OperationService {
- rpc Get (GetOperationRequest) returns (operation.Operation) {}
-}
-
-message GetOperationRequest {
- string operation_id = 1 [(required) = true, (length) = "<=50"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile.proto
deleted file mode 100644
index 8fe56d289d..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa;mfa";
-option java_outer_classname = "PTP";
-
-enum VerificationResult {
- VERIFICATION_RESULT_UNSPECIFIED = 0;
- // The verification succeeded.
- OK = 1;
- // The TOTP code sent by the client does not match the code generated on the server side.
- CODE_MISMATCH = 2;
- // The TOTP code has been used already (possibly a reuse attack).
- CODE_REUSE = 3;
- // Blocked due to brute force.
- TOO_MANY_REQUESTS = 4;
- // No TOTP configured for this user.
- NOT_SET = 5;
- // The verification succeeded, but item >= inner window size (and item < outer window size).
- NEED_MORE_CODES = 6;
-}
-
-// Default is SHA1.
-enum HashAlgorithm {
- HASH_ALGORITHM_UNSPECIFIED = 0;
- SHA1 = 1;
- SHA256 = 2;
- SHA512 = 3;
-}
-
-message TotpProfile {
- // TOTP profile ID.
- string id = 1;
- // The user that this password belongs to.
- string subject_id = 2;
- // The algorithm used to calculate the hash.
- HashAlgorithm algorithm = 3;
- // The length of the TOTP code (6-8).
- int64 digits = 4;
- google.protobuf.Timestamp created_at = 5;
- // Additional TOTP parameters.
- TotpProfileOptions options = 6;
- // True, if the user has ever been successfully verified.
- bool active = 7;
-}
-
-// Additional TOTP parameters.
-message TotpProfileOptions {
- // The period that a TOTP code will be valid for.
- google.protobuf.Duration time_step = 1 [(value) = "10s-10m"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile_service.proto
deleted file mode 100644
index 7bed0bd3b3..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa/totp_profile_service.proto
+++ /dev/null
@@ -1,125 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.mfa;
-
-import "google/protobuf/empty.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/mfa/totp_profile.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/mfa;mfa";
-option java_outer_classname = "PTPS";
-
-// A set of methods for managing time-based one time passwords (TOTP).
-// The user credentials should be passed in the authorization header.
-service TotpProfileService {
-
- // Returns the TOTP profile for the user.
- rpc Get (google.protobuf.Empty) returns (TotpProfile) {}
-
- // Creates a new TOTP profile for the user. This method will fail, if the user
- // already has an active TOTP profile.
- rpc Create (CreateTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "CreateTotpProfileMetadata"
- response: "CreateTotpProfileResponse"
- };
- }
-
- // Deletes the TOTP profile for the user.
- rpc Delete (DeleteTotpProfileRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteTotpProfileMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Retrieves the list of Operations for the specified TOTP profile.
- rpc ListOperations (ListTotpProfileOperationsRequest) returns (ListTotpProfileOperationsResponse) {}
-
- // Verifies user-supplied TOTP value. See https://tools.ietf.org/html/rfc6238#section-5.2 for the reference.
- rpc Verify (VerifyTotpRequest) returns (VerifyTotpResponse) {}
-
- // Returns the TOTP profile for the specified user.
- // This method requires `iam.totpProfiles.manage` permission.
- rpc GetForSubject (GetTotpProfileForSubjectRequest) returns (TotpProfile) {}
-
- // Deletes the TOTP profile for the specified user.
- // This method requires `iam.totpProfiles.manage` permission.
- rpc DeleteForSubject (DeleteTotpProfileForSubjectRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteTotpProfileForSubjectMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
-}
-
-message CreateTotpProfileRequest {
- // The algorithm used to calculate the hash (SHA1, SHA256 or SHA512). Default is SHA1.
- HashAlgorithm algorithm = 1;
- // The length of the TOTP code (6, 7 or 8). Default is 6.
- int64 digits = 2 [(value) = "0,6,7,8"];
- // The default value for time step is 30 seconds.
- TotpProfileOptions options = 3;
-}
-
-message CreateTotpProfileMetadata {
- string subject_id = 1;
-}
-
-message CreateTotpProfileResponse {
- // The secret parameter is an arbitrary key value encoded in Base32 according
- // to RFC https://tools.ietf.org/html/rfc3548.
- string secret = 1 [(sensitive) = true];
- // The issuer parameter is a string value indicating the provider
- // or service this account is associated with.
- string issuer = 2;
- // Newly created TOTP profile.
- TotpProfile totp_profile = 3;
-}
-
-message GetTotpProfileForSubjectRequest {
- string subject_id = 1 [(required) = true];
-}
-
-message DeleteTotpProfileRequest {
- int64 code = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_REMOVE, (value) = ">0"];
-}
-
-message DeleteTotpProfileForSubjectRequest {
- string subject_id = 1 [(required) = true];
-}
-
-message DeleteTotpProfileMetadata {
- string subject_id = 1;
-}
-
-message DeleteTotpProfileForSubjectMetadata {
- string subject_id = 1;
-}
-
-message ListTotpProfileOperationsRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListTotpProfileOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
-
-message VerifyTotpRequest {
- int64 code = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_REMOVE, (value) = ">0"];
-}
-
-message VerifyTotpResponse {
- VerificationResult result = 1;
- // HTTP-header Set-Cookie for End-User with required per-service cookies, e.g. yc_session.
- // See also yandex/cloud/priv/oauth/v1/session_service.proto specification.
- repeated string set_cookie_header = 2 [(sensitive) = true];
- google.protobuf.Timestamp retry_at = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client.proto
deleted file mode 100644
index f214bfd2d3..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POAC";
-
-message OAuthClient {
- string id = 1;
- string name = 2;
- repeated string permission_ids = 3;
- string client_secret_sha256 = 4;
- repeated string redirect_uris = 5;
- repeated string scopes = 6;
- repeated string auto_approve_scopes = 7;
- repeated string authorized_grant_types = 8;
- string federation_id = 9;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client_service.proto
deleted file mode 100644
index 98b0aef4eb..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_client_service.proto
+++ /dev/null
@@ -1,99 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/oauth_client.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POACS";
-
-service OAuthClientService {
- rpc Get (GetOAuthClientRequest) returns (OAuthClient);
-
- rpc List (ListOAuthClientsRequest) returns (ListOAuthClientsResponse);
-
- rpc Create (CreateOAuthClientRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "OAuthClient"
- metadata: "CreateOAuthClientMetadata"
- };
- }
-
- rpc Update (UpdateOAuthClientRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "OAuthClient"
- metadata: "UpdateOAuthClientMetadata"
- };
- }
-
- rpc Delete (DeleteOAuthClientRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "google.protobuf.Empty"
- metadata: "DeleteOAuthClientMetadata"
- };
- }
-}
-
-message ListOAuthClientsRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListOAuthClientsResponse {
- repeated OAuthClientListView oauth_clients = 1;
- string next_page_token = 2;
-}
-
-message GetOAuthClientRequest {
- string oauth_client_id = 1 [(required) = true];
-}
-
-message OAuthClientListView {
- string id = 1;
- string name = 2;
-}
-
-message CreateOAuthClientMetadata {
- string oauth_client_id = 1;
-}
-
-message UpdateOAuthClientMetadata {
- string oauth_client_id = 1;
-}
-
-message DeleteOAuthClientMetadata {
- string oauth_client_id = 1;
-}
-
-message CreateOAuthClientRequest {
- string oauth_client_id = 1 [(required) = true];
- string name = 2 [(required) = true];
- repeated string permission_ids = 3 [(size) = "<=10000", (length) = "<=255"];
- string client_secret_sha256 = 4 [(length) = "<=64"];
- repeated string redirect_uris = 5 [(size) = "<=1000", (length) = "<=1000"];
- repeated string scopes = 6 [(size) = "<=1000", (length) = "<=255"];
- repeated string auto_approve_scopes = 7 [(size) = "<=1000", (length) = "<=255"];
- repeated string authorized_grant_types = 8[(size) = "<=1000", (length) = "<=255"];
- string federation_id = 9 [(length) = "<=255"];
-}
-
-message UpdateOAuthClientRequest {
- string oauth_client_id = 1 [(required) = true];
- google.protobuf.FieldMask update_mask = 2;
- string name = 3;
- repeated string permission_ids = 4 [(size) = "<=10000", (length) = "<=255"];
- string client_secret_sha256 = 5 [(length) = "<=64"];
- repeated string redirect_uris = 6 [(size) = "<=1000", (length) = "<=1000"];
- repeated string scopes = 7 [(size) = "<=1000", (length) = "<=255"];
- repeated string auto_approve_scopes = 8 [(size) = "<=1000", (length) = "<=255"];
- repeated string authorized_grant_types = 9 [(size) = "<=1000", (length) = "<=255"];
- string federation_id = 10 [(length) = "<=255"];
-}
-
-message DeleteOAuthClientRequest {
- string oauth_client_id = 1 [(required) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope.proto
deleted file mode 100644
index f2080b0570..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope.proto
+++ /dev/null
@@ -1,13 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POAS";
-
-message OAuthScope {
- string id = 1;
- bool is_system = 2;
- string service = 3;
- repeated string permission_ids = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope_service.proto
deleted file mode 100644
index 1492257e48..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/oauth_scope_service.proto
+++ /dev/null
@@ -1,90 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/oauth_scope.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POASS";
-
-service OAuthScopeService {
- rpc Get (GetOAuthScopeRequest) returns (OAuthScope);
-
- rpc List (ListOAuthScopesRequest) returns (ListOAuthScopesResponse);
-
- rpc Create (CreateOAuthScopeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "OAuthScope"
- metadata: "CreateOAuthScopeMetadata"
- };
- }
-
- rpc Update (UpdateOAuthScopeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "OAuthScope"
- metadata: "UpdateOAuthScopeMetadata"
- };
- }
-
- rpc Delete (DeleteOAuthScopeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "google.protobuf.Empty"
- metadata: "DeleteOAuthScopeMetadata"
- };
- }
-}
-
-message ListOAuthScopesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListOAuthScopesResponse {
- repeated OAuthScopeListView oauth_scopes = 1;
- string next_page_token = 2;
-}
-
-message GetOAuthScopeRequest {
- string oauth_scope_id = 1 [(required) = true];
-}
-
-message OAuthScopeListView {
- string id = 1;
- bool is_system = 2;
- string service = 3;
-}
-
-message CreateOAuthScopeMetadata {
- string oauth_scope_id = 1;
-}
-
-message UpdateOAuthScopeMetadata {
- string oauth_scope_id = 1;
-}
-
-message DeleteOAuthScopeMetadata {
- string oauth_scope_id = 1;
-}
-
-message CreateOAuthScopeRequest {
- string oauth_scope_id = 1 [(required) = true];
- bool is_system = 2;
- string service = 3 [(required) = true];
- repeated string permission_ids = 4 [(size) = "<=10000", (length) = "<=255"];
-}
-
-message UpdateOAuthScopeRequest {
- string oauth_scope_id = 1 [(required) = true];
- google.protobuf.FieldMask update_mask = 2;
- bool is_system = 3;
- string service = 4;
- repeated string permission_ids = 5 [(size) = "<=10000", (length) = "<=255"];
-}
-
-message DeleteOAuthScopeRequest {
- string oauth_scope_id = 1 [(required) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/openapi-meta.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/openapi-meta.yaml
deleted file mode 100644
index 31cdeb0888..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/openapi-meta.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-info:
- title: Iam
- version: v1
- description: TODO service description
-docURLPrefix: "https://doc.cloud.yandex.ru/iaas/v1"
-docName: "FIXME-DOC-NAME-NOT-DEFINED-IN-OPEN-API-META-YAML" \ No newline at end of file
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/operation_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/operation_service.proto
deleted file mode 100644
index 94c4c74c50..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/operation_service.proto
+++ /dev/null
@@ -1,20 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POS";
-
-service OperationService {
- rpc Get (GetOperationRequest) returns (operation.Operation) {
- option (google.api.http) = { get: "/iam/v1/operations/{operation_id}" };
- }
-}
-
-message GetOperationRequest {
- string operation_id = 1 [(required) = true, (length) = "<=50"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login.proto
deleted file mode 100644
index 41608e1bf9..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POL";
-
-message OsLoginInfo {
- string subject_id = 1;
- string os_login = 2;
- int64 uid = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login_service.proto
deleted file mode 100644
index 15e5cd8112..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/os_login_service.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/os_login.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "POLS";
-
-service OsLoginService {
- rpc Get (GetOsLoginRequest) returns (OsLoginInfo);
-
- rpc Update (UpdateOsLoginRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateOsLoginMetadata"
- response: "OsLoginInfo"
- };
- }
-}
-
-message GetOsLoginRequest {
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- oneof subject {// use calling subject if not specified
- string subject_id = 2 [(length) = "<=50"];
- string os_login = 3 [(length) = "<=32"];
- int64 uid = 4;
- }
-}
-
-message UpdateOsLoginRequest {
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- string subject_id = 2 [(length) = "<=50"]; // use calling subject if not specified
- google.protobuf.FieldMask update_mask = 3;
- string os_login = 4 [(length) = "<=32"];
- int64 uid = 5;
-}
-
-message UpdateOsLoginMetadata {
- string cloud_id = 1;
- string subject_id = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission.proto
deleted file mode 100644
index 49ab3be554..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission.proto
+++ /dev/null
@@ -1,11 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PPE";
-
-message Permission {
- string id = 1;
- string stage = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_service.proto
deleted file mode 100644
index d277c9c3dc..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_service.proto
+++ /dev/null
@@ -1,80 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/permission.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PPES";
-
-service PermissionService {
- rpc Get (GetPermissionRequest) returns (Permission);
-
- rpc List (ListPermissionsRequest) returns (ListPermissionsResponse);
-
- rpc Create (CreatePermissionRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "Permission"
- metadata: "CreatePermissionMetadata"
- };
- }
-
- rpc Update (UpdatePermissionRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "Permission"
- metadata: "UpdatePermissionMetadata"
- };
- }
-
- rpc Delete (DeletePermissionRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "google.protobuf.Empty"
- metadata: "DeletePermissionMetadata"
- };
- }
-}
-
-message CreatePermissionMetadata {
- string permission_id = 1;
-}
-
-message UpdatePermissionMetadata {
- string permission_id = 1;
-}
-
-message DeletePermissionMetadata {
- string permission_id = 1;
-}
-
-message CreatePermissionRequest {
- string permission_id = 1 [(required) = true];
- string stage = 2 [(required) = true];
-}
-
-message UpdatePermissionRequest {
- string permission_id = 1 [(required) = true];
- google.protobuf.FieldMask update_mask = 2;
- string stage = 3;
-}
-
-message DeletePermissionRequest {
- string permission_id = 1 [(required) = true];
-}
-
-message GetPermissionRequest {
- string permission_id = 1 [(required) = true];
-}
-
-message ListPermissionsRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListPermissionsResponse {
- repeated Permission permissions = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage.proto
deleted file mode 100644
index 398c1b6f5f..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage.proto
+++ /dev/null
@@ -1,36 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PPS";
-
-message PermissionStage {
- string id = 1;
- string description = 2;
-}
-
-message SetAllPermissionStagesRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message SetPermissionStagesRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- repeated string permission_stage_ids = 2;
-}
-
-message SetPermissionStagesMetadata {
- string resource_id = 1;
-}
-
-message UpdatePermissionStagesRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- repeated string add_permission_stage_ids = 2;
- repeated string remove_permission_stage_ids = 3;
-}
-
-message UpdatePermissionStagesMetadata {
- string resource_id = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage_service.proto
deleted file mode 100644
index a3f494a2bd..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permission_stage_service.proto
+++ /dev/null
@@ -1,70 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/permission_stage.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PPSS";
-
-service PermissionStageService {
- rpc Get (GetPermissionStageRequest) returns (PermissionStage);
-
- rpc List (ListPermissionStagesRequest) returns (ListPermissionStagesResponse);
-
- rpc Create (CreatePermissionStageRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "PermissionStage"
- metadata: "CreatePermissionStageMetadata"
- };
- }
-
- rpc Delete (DeletePermissionStageRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "google.protobuf.Empty"
- metadata: "DeletePermissionMetadata"
- };
- }
-}
-
-message CreatePermissionStageMetadata {
- string permission_stage_id = 1;
-}
-
-message UpdatePermissionStageMetadata {
- string permission_stage_id = 1;
-}
-
-message DeletePermissionStageMetadata {
- string permission_stage_id = 1;
-}
-
-message CreatePermissionStageRequest {
- string permission_stage_id = 1 [(required) = true];
-}
-
-message UpdatePermissionStageRequest {
- string permission_stage_id = 1 [(required) = true];
-}
-
-message DeletePermissionStageRequest {
- string permission_stage_id = 1 [(required) = true];
-}
-
-message GetPermissionStageRequest {
- string permission_stage_id = 1 [(required) = true];
-}
-
-message ListPermissionStagesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
- string filter = 3 [(length) = "<=1000"];
-}
-
-message ListPermissionStagesResponse {
- repeated PermissionStage permission_stages = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permissions.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permissions.yaml
deleted file mode 100644
index 2279852711..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/permissions.yaml
+++ /dev/null
@@ -1,1481 +0,0 @@
-permissions:
-
- iam.permissionStages.get:
- description: Get IAM permission stages
- stage: GA
- visibility: internal
- iam.permissionStages.create:
- description: Create IAM permission stages
- stage: GA
- visibility: internal
-
- iam.resourceTypes.get:
- description: Get IAM resource types
- stage: GA
- visibility: internal
- iam.resourceTypes.create:
- description: Create IAM resource types
- stage: GA
- visibility: internal
- iam.resourceTypes.update:
- description: Update IAM resource types
- stage: GA
- visibility: internal
- iam.resourceTypes.listRestrictions:
- description: List assignment of restrictions on all resources of particular type
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
- iam.resourceTypes.updateRestrictions:
- description: Manage assignment of restrictions on all resources of particular type
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.permissions.get:
- description: Get IAM permissions
- stage: GA
- visibility: internal
- iam.permissions.create:
- description: Create IAM permissions
- stage: GA
- visibility: internal
- iam.permissions.update:
- description: Update IAM permissions
- stage: GA
- visibility: internal
-
- iam.roles.get:
- description: Get IAM roles
- stage: GA
- visibility: internal
- iam.roles.create:
- description: Create IAM roles
- stage: GA
- visibility: internal
- iam.roles.update:
- description: Update IAM roles
- stage: GA
- visibility: internal
-
- iam.restrictionTypes.get:
- description: Get IAM restriciton types
- stage: GA
- visibility: internal
- resourceType: iam.restrictionType
- iam.restrictionTypes.create:
- description: Create IAM restriction types
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
- iam.restrictionTypes.update:
- description: Update IAM restriction types
- stage: GA
- visibility: internal
- resourceType: iam.restrictionType
- iam.restrictionTypes.listAccessBindings:
- description: List access bindings on restriction type
- stage: GA
- visibility: internal
- resourceType: iam.restrictionType
- iam.restrictionTypes.updateAccessBindings:
- description: Update access bindings on restriction type
- stage: GA
- visibility: internal
- resourceType: iam.restrictionType
- iam.restrictionTypes.use:
- description: Manage particular type of restriction on resources
- stage: GA
- visibility: internal
- resourceType: iam.restrictionType
-
- iam.oauthScopes.get:
- description: Get OAuth scopes
- stage: GA
- visibility: internal
- iam.oauthScopes.create:
- description: Create OAuth scopes
- stage: GA
- visibility: internal
- iam.oauthScopes.update:
- description: Update OAuth scopes
- stage: GA
- visibility: internal
-
- iam.oauthClients.get:
- description: Get OAuth clients
- stage: GA
- visibility: internal
- iam.oauthClients.create:
- description: Create OAuth clients
- stage: GA
- visibility: internal
- iam.oauthClients.update:
- description: Update OAuth clients
- stage: GA
- visibility: internal
-
- iam.accessBinding.delete:
- description: Delete role assignment
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- iam.accessBindings.list:
- description: List all role assignments
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- - DELETING
- iam.accessBindings.manageOwners:
- description: Manage owner roles in a cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- cloud:
- status:
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- iam.accessBindings.set:
- description: Create role assignment
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- iam.accessBindings.update:
- description: Create role assignment
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- iam.accessBindings.changeTopLevelResource:
- description: Change top level resource of access bindings
- stage: GA
- visibility: internal
- iam.accessBindings.deleteByTopLevelResource:
- description: Delete access bindings by top level resource
- stage: GA
- visibility: internal
-
- iam.cloud.update:
- # TODO: rename
- description: Edits an cloud
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
-
- iam.cloudOperations.list:
- description: List cloud operations
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.cloudUser.create:
- description: Create user in cloud
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- iam.cloudUser.delete:
- description: Deletes a user
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- iam.cloudUser.get:
- description: Get one user in a given cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.cloudUsers.list:
- description: List users in a cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- iam.cloudUsers.superList:
- description: List users in a cloud with disabled user listing
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.clouds.delete:
- description: Deletes an cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- iam.clouds.get:
- description: Returns a single cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - CREATING
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: resource-manager.cloud
- iam.clouds.getSettings:
- description: Returns a single cloud settings
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - CREATING
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- iam.clouds.updateSettings:
- description: Updates a single cloud settings
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- cloud:
- status:
- - CREATING
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.clouds.list:
- description: List clouds where current user has membership
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.clouds.setDefaultZone:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- iam.clouds.setUserListingSetting:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: resource-manager.cloud
-
- iam.federations.create:
- description: ''
- stage: IAM_FOLDER_SAML_FEDERATIONS
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: resource-manager.folder
- iam.federations.createCertificate:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.federation
- iam.federations.createUser:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.federation
- iam.federations.delete:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.federation
- iam.federations.deleteCertificate:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.federationCertificate
- iam.federations.deleteUser:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- iam.federations.get:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.federation
- iam.federations.getCertificate:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.federationCertificate
- iam.federations.list:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: resource-manager.folder
- iam.federations.listCertificate:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.federation
- iam.federations.update:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.federation
- iam.federations.updateCertificate:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.federationCertificate
-
- iam.federatedUsers.crossCloudBindings:
- description: ''
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.folderOperations.list:
- description: List folder operations
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.folders.create:
- description: Create folder
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: resource-manager.cloud
- iam.folders.delete:
- description: Deletes a folder
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: resource-manager.folder
- iam.folders.get:
- description: Returns a single folder
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: resource-manager.folder
- iam.folders.getSettings:
- description: Returns a single folder settings
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - CREATING
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.folders.updateSettings:
- description: Updates a single folder settings
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- cloud:
- status:
- - CREATING
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.folders.list:
- description: List folders in an cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: resource-manager.cloud
- iam.folders.update:
- description: Update folder
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: resource-manager.folder
-
- iam.operations.get:
- description: Get operation
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- iam.operations.list:
- description: Get operation
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.roles.list:
- description: Get the list of global roles in an cloud
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.serviceAccounts.create:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: resource-manager.folder
- iam.serviceAccounts.createForHost:
- description: ''
- stage: GA
- visibility: public
- iam.serviceAccounts.delete:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.getAccessKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.createAccessKey:
- description: Create access key for service account
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.serviceAccount
- iam.serviceAccounts.createAccessKeyWithPrefix:
- description: Create access key for service account with regional prefix
- stage: IAM_ACCESS_KEY_PREFIX
- visibility: internal
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.serviceAccount
- iam.serviceAccounts.updateAccessKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.deleteAccessKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.getApiKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.createApiKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.serviceAccount
- iam.serviceAccounts.updateApiKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.deleteApiKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.getSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.createSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.serviceAccount
- iam.serviceAccounts.deleteSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.serviceAccount
- iam.userAccounts.getSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.userAccount
- iam.userAccounts.createSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.userAccount
- iam.userAccounts.deleteSshKey:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.userAccount
- iam.userAccounts.getSettings:
- description: 'Чтение userSettings пользователя'
- stage: GA
- visibility: public
- resourceType: iam.userAccount
- iam.serviceAccounts.get:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.issueToken:
- description: ''
- stage: GA
- visibility: public
- resourceType: iam.serviceAccount
- iam.serviceAccounts.issueCookie:
- description: ''
- stage: GA
- visibility: internal
- resourceType: iam.serviceAccount
- iam.serviceAccounts.issueTokenForInstance:
- description: ''
- stage: GA
- visibility: public
- resourceType: iam.serviceAccount
- iam.serviceAccounts.issueTokenForRestrictedAlgorithm:
- description: ''
- stage: GA
- visibility: internal
- resourceType: iam.serviceAccount
- iam.serviceAccounts.list:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: resource-manager.folder
- iam.serviceAccounts.listAccessKeys:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.accessKeys.listByFingerprint:
- description: List access keys by secret key fingerprint
- stage: GA
- visibility: internal
- iam.serviceAccounts.listApiKeys:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.serviceAccount
- iam.serviceAccounts.update:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.serviceAccount
- iam.serviceAccounts.use:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- iam.serviceAccounts.crossCloudBindings:
- description: ''
- stage: GA
- visibility: internal
- resourceType: iam.serviceAccount
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- iam.serviceAccounts.listAccessBindings:
- description: ''
- stage: GA
- visibility: public
- resourceType: iam.serviceAccount
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- - DELETING
- iam.serviceAccounts.updateAccessBindings:
- description: ''
- stage: GA
- visibility: public
- resourceType: iam.serviceAccount
- allowedWhen:
- restrictions:
- - freeTier
- - billSuspend
- cloud:
- status:
- - BLOCKED_BY_BILLING
- - CREATING
- - ACTIVE
- iam.tokenKeys.create:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.tokenKeys.update:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.tokenKeys.delete:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.tokenKeys.get:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
-
- iam.authorizedKeys.create:
- description: Create authorized key
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.authorizedKeys.update:
- description: Update authorized key
- stage: GA
- visibility: public
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.authorizedKeys.delete:
- description: Delete authorized key
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.authorizedKeys.get:
- description: Get authorized key
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
- iam.authorizedKeys.listByFingerprint:
- description: List authorized keys by public key fingerprint
- stage: GA
- visibility: internal
-
- iam.userAccounts.createAccessKey:
- description: Create user access key
- stage: GA
- visibility: internal
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.userAccount
- iam.userAccounts.createAccessKeyWithPrefix:
- description: Create user access key with regional prefix
- stage: IAM_ACCESS_KEY_PREFIX
- visibility: internal
- allowedWhen:
- cloud:
- status:
- - ACTIVE
- resourceType: iam.userAccount
- iam.userAccounts.deleteAccessKey:
- description: Delete user access key
- stage: GA
- visibility: internal
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.userAccount
- iam.userAccounts.getAccessKey:
- description: Get user access key
- stage: GA
- visibility: internal
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.userAccount
- iam.userAccounts.listAccessKeys:
- description: List user access keys
- stage: GA
- visibility: internal
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
- resourceType: iam.userAccount
- iam.userAccounts.updateAccessKey:
- description: Update user access key
- stage: GA
- visibility: internal
- allowedWhen:
- restrictions:
- - billSuspend
- cloud:
- status:
- - ACTIVE
- - BLOCKED_BY_BILLING
- - DELETING
- resourceType: iam.userAccount
-
- iam.userAccounts.createManaged:
- description: 'Create a user account that is managed by an external system.'
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
-
- iam.userAccounts.delete:
- description: 'Delete a user account.'
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
-
- iam.userAccounts.get:
- description: ''
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.subjects.issueCookie:
- description: 'Issue cookie for subject'
- stage: GA
- visibility: internal
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
-
- iam.subjects.issueToken:
- description: 'Issue token for subject'
- stage: GA
- visibility: internal
- resourceType:
- oneOf: [iam.serviceAccount, iam.userAccount]
-
- iam.userAccounts.issueCookie:
- description: 'Issue cookie for user account'
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.userAccounts.issueToken:
- description: 'Issue token for user account'
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.userAccounts.issueTokenFromJwt:
- description: 'Issue token for user account from JWT'
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.userAccounts.createTotpProfile:
- description: 'Create TOTP profile for user account'
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.userAccounts.presignUrl:
- description: 'Create signature for storage URL'
- stage: GA
- visibility: internal
- resourceType: iam.userAccount
-
- iam.userOperations.list:
- description: List user operations
- stage: GA
- visibility: public
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.resourceTypes.membership:
- description: Special permission granting membership in a top-level resource
- stage: GA
- visibility: public
-
- iam.resourceTypes.listMemberships:
- description: View top-level resource members and member resources
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.resourceTypes.listAccessBindings:
- description: List access bindings on resource type
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.resourceTypes.updateAccessBindings:
- description: Update access bindings on resource type
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.gizmo.listAccessBindings:
- description: List gizmo access bindings
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.gizmo.updateAccessBindings:
- description: Update gizmo access bindings
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.root.listAccessBindings:
- description: List root access bindings
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.root.updateAccessBindings:
- description: Update root access bindings
- stage: GA
- visibility: internal
- resourceType: iam.resourceType
-
- iam.subjects.getOsLogin:
- description: Get OS login information of a subject
- stage: GA
- visibility: internal
- resourceType: iam.subject
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.subjects.updateOsLogin:
- description: Update OS login information of a subject
- stage: GA
- visibility: internal
- resourceType: iam.subject
- allowedWhen:
- cloud:
- status:
- - ACTIVE
-
- iam.quotas.get:
- description: Get IAM quotas
- stage: GA
- visibility: public
- resourceType: resource-manager.cloud
- allowedWhen:
- restrictions:
- - billSuspend
- - fraud
- - deletingContainer
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.quotas.updateLimit:
- description: Update IAM quotas
- stage: GA
- visibility: internal
- resourceType: resource-manager.cloud
- allowedWhen:
- restrictions:
- - billSuspend
- cloud:
- status:
- - BLOCKED
- - BLOCKED_BY_BILLING
- - ACTIVE
- - DELETING
-
- iam.internal.diagnostics:
- description: Usage of intenal diagnostic tools
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
-
- iam.totpProfiles.manage:
- description: ''
- stage: GA
- visibility: internal
- resourceType: iam.totpProfile
-
- iam.invitees.getOrCreate:
- description: Create (or get if exists) invitees
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
-
- iam.subjects.merge:
- description: Merge subjects
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
-
- iam.subjects.delete:
- description: Delete subjects
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quota_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quota_service.proto
deleted file mode 100644
index b425b36e1d..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quota_service.proto
+++ /dev/null
@@ -1,17 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/empty.proto";
-import "yandex/cloud/priv/quota/quota.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PQS";
-
-service QuotaService {
- rpc Get (quota.GetQuotaRequest) returns (quota.Quota);
-
- rpc Update (quota.UpdateQuotaMetricRequest) returns (google.protobuf.Empty);
-
- rpc GetDefault (quota.GetQuotaDefaultRequest) returns (quota.GetQuotaDefaultResponse);
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quotas.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quotas.yaml
deleted file mode 100644
index 8f3a2d1678..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/quotas.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-quotas:
- - iam.accessBindings.count
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type.proto
deleted file mode 100644
index 59c74b882e..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type.proto
+++ /dev/null
@@ -1,11 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRST";
-
-message ResourceType {
- string id = 1;
- string access_bindings_listing_permission = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type_service.proto
deleted file mode 100644
index a04190169c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resource_type_service.proto
+++ /dev/null
@@ -1,105 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/iam/v1/resource_type.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRSTS";
-
-service ResourceTypeService {
- rpc Get (GetResourceTypeRequest) returns (ResourceType);
-
- rpc List (ListResourceTypesRequest) returns (ListResourceTypeResponse);
-
- rpc Create (CreateResourceTypeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "ResourceType"
- metadata: "CreateResourceTypeMetadata"
- };
- }
-
- rpc Update (UpdateResourceTypeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "ResourceType"
- metadata: "UpdateResourceTypeMetadata"
- };
- }
-
- rpc Delete (DeleteResourceTypeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "ResourceType"
- metadata: "DeleteResourceTypeMetadata"
- };
- }
-
- rpc ListAccessBindings (ListResourceTypeAccessBindingsRequest) returns (access.ListAccessBindingsResponse);
-
- rpc UpdateAccessBindings (UpdateResourceTypeAccessBindingsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateResourceTypeAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message CreateResourceTypeMetadata {
- string resource_type = 1;
-}
-
-message UpdateResourceTypeMetadata {
- string resource_type = 1;
-}
-
-message DeleteResourceTypeMetadata {
- string resource_type = 1;
-}
-
-message CreateResourceTypeRequest {
- string resource_type = 1 [(required) = true];
- string access_bindings_listing_permission = 2;
-}
-
-message UpdateResourceTypeRequest {
- string resource_type = 1 [(required) = true];
- google.protobuf.FieldMask update_mask = 2;
- string access_bindings_listing_permission = 3;
-}
-
-message DeleteResourceTypeRequest {
- string resource_type = 1 [(required) = true];
-}
-
-message GetResourceTypeRequest {
- string resource_type = 1 [(required) = true];
-}
-
-message ListResourceTypesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message ListResourceTypeResponse {
- repeated ResourceType resource_types = 1;
- string next_page_token = 2;
-}
-
-message ListResourceTypeAccessBindingsRequest {
- string resource_type = 1 [(required) = true, (length) = "<=100"];
- int64 page_size = 2 [(value) = "<=1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message UpdateResourceTypeAccessBindingsRequest {
- string resource_type = 1 [(required) = true, (length) = "<=100"];
- repeated access.AccessBindingDelta access_binding_deltas = 2 [(size) = ">0"];
-}
-
-message UpdateResourceTypeAccessBindingsMetadata {
- string resource_type = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resources.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resources.yaml
deleted file mode 100644
index 885c82a3d1..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/resources.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-resources:
- iam.resourceType:
- parents: [root]
- iam.gizmo:
- parents: [root]
- iam.subject:
- parents: [root]
- iam.restrictionType:
- parents: [root]
- iam.serviceAccount:
- parents: [iam.subject, resource-manager.folder]
- accessBindingsListingPermission: iam.serviceAccounts.listAccessBindings
- iam.userAccount:
- parents: [iam.subject]
- iam.accessKey:
- parents: [iam.subject]
- iam.authorizedKey:
- parents: [iam.subject]
- iam.apiKey:
- parents: [iam.serviceAccount]
- iam.federationCertificate:
- parents: [iam.federation]
- iam.federation:
- parents: [resource-manager.folder]
- iam.totpProfile:
- parents: [iam.subject]
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction.proto
deleted file mode 100644
index 6b9d7aaf9d..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction.proto
+++ /dev/null
@@ -1,16 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/iam/v1/restriction_type.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRN";
-
-message Restriction {
- RestrictionKind restriction_kind = 1;
- string restriction_type_id = 2;
- google.protobuf.Timestamp added_at = 3;
- string added_by = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_service.proto
deleted file mode 100644
index 19b5efa97c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_service.proto
+++ /dev/null
@@ -1,89 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/restriction.proto";
-import "yandex/cloud/priv/iam/v1/restriction_type.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRNS";
-
-service RestrictionService {
- rpc List (ListRestrictionsRequest) returns (ListRestrictionsResponse);
-
- rpc Add (AddRestrictionRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "AddRestrictionMetadata"
- response: "Restriction"
- };
- }
-
- rpc Remove (RemoveRestrictionRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "RemoveRestrictionMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc RemoveAll (RemoveAllRestrictionsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "RemoveAllRestrictionsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message ListRestrictionsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- RestrictionKind restriction_kind = 3;
- int64 page_size = 4 [(value) = "<=1000"];
- string page_token = 5 [(length) = "<=100"];
-}
-
-message ListRestrictionsResponse {
- repeated Restriction restrictions = 1;
- string next_page_token = 2;
-}
-
-message AddRestrictionRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- // If set, perform operation on behalf of subject with this id.
- string on_behalf_of_subject_id = 3 [(length) = "<=50"];
- string restriction_type_id = 4 [(required) = true, (length) = "<=100"];
-}
-
-message AddRestrictionMetadata {
- string resource_id = 1;
- string resource_type = 2;
- string on_behalf_of_subject_id = 3;
-}
-
-message RemoveRestrictionRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
- // If set, perform operation on behalf of subject with this id.
- string on_behalf_of_subject_id = 3 [(length) = "<=50"];
- string restriction_type_id = 4 [(required) = true, (length) = "<=100"];
-}
-
-message RemoveRestrictionMetadata {
- string resource_id = 1;
- string resource_type = 2;
- string on_behalf_of_subject_id = 3;
- string restriction_type_id = 4;
-}
-
-message RemoveAllRestrictionsRequest {
- string resource_id = 1 [(required) = true, (length) = "<=50"];
- string resource_type = 2 [(required) = true, (length) = "<=100"];
-}
-
-message RemoveAllRestrictionsMetadata {
- string resource_id = 1;
- string resource_type = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type.proto
deleted file mode 100644
index 59de0f83d8..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/duration.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRNT";
-
-enum RestrictionKind {
- RESTRICTION_KIND_UNSPECIFIED = 0;
- BLOCK_PERMISSIONS = 1;
-}
-
-message RestrictionType {
- string id = 1;
- RestrictionKind restriction_kind = 2;
-
- oneof type {
- BlockPermissions block_permissions = 3;
- }
-
- message BlockPermissions {
- string role_mask = 1;
- bool direct_mask = 2;
- repeated string services_to_stop = 3;
- repeated string resources_to_stop = 4;
- google.protobuf.Duration stop_delay = 5;
- google.protobuf.Duration deletion_initiation_interval = 6;
- google.protobuf.Duration deletion_delay = 7;
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type_service.proto
deleted file mode 100644
index 845e9aeb70..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/restriction_type_service.proto
+++ /dev/null
@@ -1,114 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/iam/v1/restriction_type.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRNTS";
-
-service RestrictionTypeService {
- rpc Get (GetRestrictionTypeRequest) returns (RestrictionType);
-
- rpc List (ListRestrictionTypesRequest) returns (ListRestrictionTypesResponse);
-
- rpc Create (CreateRestrictionTypeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "CreateRestrictionTypeMetadata"
- response: "RestrictionType"
- };
- }
-
- rpc Update (UpdateRestrictionTypeRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateRestrictionTypeMetadata"
- response: "RestrictionType"
- };
- }
-
- // access
-
- rpc ListAccessBindings (access.ListAccessBindingsRequest) returns (access.ListAccessBindingsResponse);
-
- rpc SetAccessBindings (access.SetAccessBindingsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "access.SetAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc UpdateAccessBindings (access.UpdateAccessBindingsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "access.UpdateAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message GetRestrictionTypeRequest {
- string restriction_type_id = 1 [(required) = true, (length) = "<=100"];
-}
-
-message ListRestrictionTypesRequest {
- RestrictionKind restriction_kind = 1;
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListRestrictionTypesResponse {
- repeated RestrictionType restriction_types = 1;
- string next_page_token = 2;
-}
-
-message CreateRestrictionTypeRequest {
- string restriction_type_id = 1 [(required) = true, (length) = "<=100"];
-
- oneof type {
- option (exactly_one) = true;
- BlockPermissions block_permissions = 2;
- }
-
- message BlockPermissions {
- string role_mask = 1 [(required) = true, (length) = "<=100"];
- bool direct_mask = 2;
- repeated string services_to_stop = 3 [(size) = "<=100", (length) = "<=100"];
- repeated string resources_to_stop = 4 [(size) = "<=100", (length) = "<=100"];
- google.protobuf.Duration stop_delay = 5;
- google.protobuf.Duration deletion_initiation_interval = 6;
- google.protobuf.Duration deletion_delay = 7;
- }
-}
-
-message CreateRestrictionTypeMetadata {
- string restriction_type_id = 1;
-}
-
-message UpdateRestrictionTypeRequest {
- string restriction_type_id = 1 [(required) = true, (length) = "<=100"];
- google.protobuf.FieldMask update_mask = 2;
-
- oneof type {
- option (exactly_one) = true;
- BlockPermissions block_permissions = 3;
- }
-
- message BlockPermissions {
- string role_mask = 1 [(length) = "<=100"];
- bool direct_mask = 2;
- repeated string services_to_stop = 3 [(size) = "<=100", (length) = "<=100"];
- repeated string resources_to_stop = 4 [(size) = "<=100", (length) = "<=100"];
- google.protobuf.Duration stop_delay = 5;
- google.protobuf.Duration deletion_initiation_interval = 6;
- google.protobuf.Duration deletion_delay = 7;
- }
-}
-
-message UpdateRestrictionTypeMetadata {
- string restriction_type_id = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role.proto
deleted file mode 100644
index 4aed76da4c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role.proto
+++ /dev/null
@@ -1,13 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PR";
-
-message Role {
- string id = 1;
- string description = 2;
- repeated string permission_ids = 3;
- bool is_system = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role_service.proto
deleted file mode 100644
index 807308a7d5..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/role_service.proto
+++ /dev/null
@@ -1,95 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/role.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRS";
-
-service RoleService {
- rpc Get (GetRoleRequest) returns (Role);
-
- rpc List (ListRolesRequest) returns (ListRolesResponse);
-
- rpc Create (CreateRoleRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "Role"
- metadata: "CreateRoleMetadata"
- };
- }
-
- rpc Update (UpdateRoleRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- response: "Role"
- metadata: "UpdateRoleMetadata"
- };
- }
-
- rpc Delete (DeleteRoleRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteRoleMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message CreateRoleMetadata {
- string role_id = 1;
-}
-
-message UpdateRoleMetadata {
- string role_id = 1;
-}
-
-message DeleteRoleMetadata {
- string role_id = 1;
-}
-
-message CreateRoleRequest {
- string role_id = 1 [(required) = true];
- bool is_system = 2;
- repeated string permission_ids = 3 [(size) = "<=10000", (length) = "<=255"];
-}
-
-message UpdateRoleRequest {
- string role_id = 1 [(required) = true];
- google.protobuf.FieldMask update_mask = 2;
- bool is_system = 3;
- repeated string permission_ids = 4 [(size) = "<=10000", (length) = "<=255"];
-}
-
-message DeleteRoleRequest {
- string role_id = 1 [(required) = true];
-}
-
-message GetRoleRequest {
- string role_id = 1 [(required) = true];
-}
-
-message ListRolesRequest {
- int64 page_size = 1 [(value) = "0-1000"];
- string page_token = 2 [(length) = "<=100"];
- string filter = 3 [(length) = "<=1000"];
-
- RoleView view = 4;
-}
-
-message ListRolesResponse {
- repeated Role roles = 1;
- string next_page_token = 2;
-}
-
-enum RoleView {
- option (cloud.api.tools.enumeration).lint_skip.unspecified_value = true;
- // Omits the permission_ids field. This is the default value.
- BASIC = 0;
-
- // Returns all fields.
- FULL = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/roles.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/roles.yaml
deleted file mode 100644
index 86e2f6a6ba..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/roles.yaml
+++ /dev/null
@@ -1,568 +0,0 @@
-role_groups:
- iam:
- name: IAM roles
-
-roles:
-
- ##############################################################################
- # Public Roles
- ##############################################################################
-
- iam.viewer:
- groups: [iam]
- name: IAM-viewer
- visibility: public
- description: ''
- scopes:
- - yc.iam.serviceAccounts.accessKey.get
- - yc.iam.serviceAccounts.apiKey.get
- - yc.iam.serviceAccounts.sshKey.get
- - yc.iam.serviceAccounts.get
- - yc.resource-manager.folders.get # ?
- - yc.resource-manager.clouds.get # ?
- permissions:
- - iam.accessBindings.list
- - iam.authorizedKeys.get
- - iam.cloudOperations.list
- - iam.cloudUser.get
- - iam.cloudUsers.list
- - iam.cloudUsers.superList
- - iam.federations.get
- - iam.folderOperations.list
- - iam.operations.{get,list}
- - iam.roles.list
- - iam.serviceAccounts.listAccessBindings
- - iam.tokenKeys.get
- - iam.userAccounts.get
- - iam.userOperations.list
- - iam.quotas.get
-
- iam.editor:
- groups: [iam]
- name: IAM-editor
- visibility: public
- description: ''
- includedRoles:
- - iam.viewer
- scopes:
- - yc.iam.serviceAccounts.manage
- - yc.resource-manager.folders.manage
- - yc.iam.serviceAccounts.use
- permissions:
- - iam.authorizedKeys.{create,delete,update}
- - iam.cloud.update
- - iam.clouds.getSettings
- - iam.clouds.updateSettings
- - iam.cloudUser.delete
- - iam.folders.getSettings
- - iam.folders.updateSettings
- - iam.tokenKeys.{create,delete,update}
- - iam.userAccounts.{create,delete}SshKey
-
- iam.admin:
- groups: [iam]
- name: IAM-admin
- visibility: public
- description: ''
- includedRoles:
- - iam.editor
- - iam.serviceAccounts.admin
- scopes:
- - yc.iam.federations.manage
- - yc.iam.accessBindings.manage
- - yc.iam.clouds.manage
- permissions:
- - iam.cloudUser.create
-
- iam.serviceAccounts.admin:
- groups: [iam]
- name: Администратор сервисных аккаунтов
- visibility: public
- description: ''
- scopes:
- - yc.iam.serviceAccounts.manage
- - yc.iam.authorizedKeys.get
- - yc.iam.serviceAccounts.accessKey.get
- - yc.iam.serviceAccounts.apiKey.get
- - yc.iam.serviceAccounts.sshKey.get
- - yc.iam.serviceAccounts.get
- - yc.iam.serviceAccounts.use
- - yc.iam.serviceAccounts.issueToken
- - yc.resource-manager.folders.get
- permissions:
- - iam.serviceAccounts.{list,update}AccessBindings
-
- iam.serviceAccounts.tokenCreator:
- groups: [iam]
- name: Роль для получения токена сервис-аккаунта, на который она назначена
- visibility: public
- description: ''
- scopes:
- - yc.iam.serviceAccounts.issueToken
-
- iam.serviceAccounts.user:
- groups: [iam]
- name: Пользователь сервисного аккаунта
- visibility: public
- description: ' специальная роль для доступа к сервисным аккаунтам'
- scopes:
- - yc.iam.serviceAccounts.get
- - yc.iam.serviceAccounts.use
- permissions: []
-
- iam.serviceAccounts.accessKeyAdmin:
- groups: [iam]
- name: Администратор access-ключей
- visibility: public
- description: ''
- permissions:
- - iam.serviceAccounts.{create,get,update,delete}AccessKey
-
- iam.serviceAccounts.apiKeyAdmin:
- groups: [iam]
- name: Администратор API-ключей
- visibility: public
- description: ''
- permissions:
- - iam.serviceAccounts.{create,get,update,delete}ApiKey
-
- iam.serviceAccounts.authorizedKeyAdmin:
- groups: [iam]
- name: Администратор авторизованных ключей
- visibility: public
- description: ''
- permissions:
- - iam.authorizedKeys.{create,get,update,delete}
-
- iam.serviceAccounts.keyAdmin:
- groups: [iam]
- name: Администратор ключей
- visibility: public
- description: ''
- includedRoles:
- - iam.serviceAccounts.accessKeyAdmin
- - iam.serviceAccounts.apiKeyAdmin
- - iam.serviceAccounts.authorizedKeyAdmin
-
- ##############################################################################
- # Support Roles
- ##############################################################################
-
- iam.support:
- groups: [iam, staff]
- name: Support IAM
- visibility: internal
- description: Role for support engineers to see access bindings and premissions
- permissions:
- - iam.accessBindings.list
- - iam.authorizedKeys.get
- - iam.cloudOperations.list
- - iam.cloudUser.get
- - iam.cloudUsers.list
- - iam.cloudUsers.superList
- - iam.clouds.{get,list}
- - iam.folderOperations.list
- - iam.folders.{get,list}
- - iam.folders.create
- - iam.operations.{get,list}
- - iam.roles.list
- - iam.serviceAccounts.getAccessKey
- - iam.serviceAccounts.getApiKey
- - iam.serviceAccounts.getSshKey
- - iam.serviceAccounts.listAccessKeys
- - iam.serviceAccounts.listApiKeys
- - iam.serviceAccounts.{get,list}
- - iam.serviceAccounts.listAccessBindings
- - iam.tokenKeys.get
- - iam.userAccounts.get
- - iam.userAccounts.getAccessKey
- - iam.userAccounts.getSettings
- - iam.userAccounts.getSshKey
- - iam.userAccounts.listAccessKeys
- - iam.userOperations.list
- - iam.federations.{get,list}
- - iam.quotas.updateLimit
- - resource-manager.clouds.listAccessBindings
- - resource-manager.clouds.listRestrictions
- - resource-manager.clouds.listPermissionStages
- - resource-manager.clouds.updatePermissionStages
- - resource-manager.folders.listAccessBindings
-
- iam.supportAdmin:
- groups: [iam, staff]
- name: Support IAM Admin
- visibility: internal
- description: Privileged Operations in IAM for support engineers
- includedRoles:
- - iam.support
- permissions: []
-
- ##############################################################################
- # On-call Roles
- ##############################################################################
-
- iam.onCall:
- groups: [iam, staff]
- name: On-call IAM
- visibility: internal
- description: Role for IAM on-call engineers to see access bindings and premissions
- includedRoles:
- - onCall
- permissions:
- - iam.accessBindings.list
- - iam.authorizedKeys.get
- - iam.cloudOperations.list
- - iam.cloudUser.get
- - iam.cloudUsers.list
- - iam.cloudUsers.superList
- - iam.clouds.{get,list}
- - iam.folderOperations.list
- - iam.folders.{get,list}
- - iam.folders.create
- - iam.operations.{get,list}
- - iam.roles.list
- - iam.serviceAccounts.getAccessKey
- - iam.serviceAccounts.getApiKey
- - iam.serviceAccounts.getSshKey
- - iam.serviceAccounts.listAccessKeys
- - iam.serviceAccounts.listApiKeys
- - iam.serviceAccounts.{get,list}
- - iam.serviceAccounts.listAccessBindings
- - iam.tokenKeys.get
- - iam.userAccounts.get
- - iam.userAccounts.getAccessKey
- - iam.userAccounts.getSettings
- - iam.userAccounts.getSshKey
- - iam.userAccounts.listAccessKeys
- - iam.userOperations.list
- - iam.federations.{get,list}
- - resource-manager.clouds.listAccessBindings
- - resource-manager.clouds.listRestrictions
- - resource-manager.clouds.listPermissionStages
- - resource-manager.clouds.updatePermissionStages
- - resource-manager.folders.listAccessBindings
-
- iam.onCallAdmin:
- groups: [iam, staff]
- name: On-call IAM Admin
- visibility: internal
- description: Privileged Operations in IAM for on-call engineers
- includedRoles:
- - iam.onCall
- - internal.iam.metaModelEditor
- permissions:
- - iam.accessBinding.delete
- - iam.accessBindings.update
- - iam.accessBindings.manageOwners
- - iam.authorizedKeys.delete
- - iam.cloudUser.delete
- - iam.clouds.delete
- - iam.folders.delete
- - iam.serviceAccounts.delete
- - iam.serviceAccounts.deleteAccessKey
- - iam.serviceAccounts.deleteApiKey
- - iam.serviceAccounts.deleteSshKey
- - iam.tokenKeys.delete
- - iam.userAccounts.{delete,update}AccessKey
- - iam.userAccounts.deleteSshKey
- - iam.quotas.updateLimit
- - iam.totpProfiles.manage
-
-
- ##############################################################################
- # Internal Roles
- ##############################################################################
-
- internal.iam.restrictedTokenCreator:
- groups: [iam]
- name: Роль для получения токена сервис-аккаунта, который лежит в folderе на который назначена эта роль
- visibility: internal
- description: ''
- permissions:
- - iam.serviceAccounts.issueTokenForRestrictedAlgorithm
-
- internal.iam.prefixedAccessKeyCreator:
- groups: [iam]
- name: Роль для создания Access Key с региональным префиксом
- visibility: internal
- description: ''
- permissions:
- - iam.serviceAccounts.createAccessKeyWithPrefix
- - iam.userAccounts.createAccessKeyWithPrefix
-
- internal.identityagent:
- groups: [iam]
- name: (Внутренняя) Агент Identity
- visibility: internal
- description: ' доступ для создания сетей и presigned URL'
- permissions:
- - billingInternal.accounts.activateReferralCode
- - billingInternal.accounts.bindCloud
- - compute.instances.{get,list}
- - computeInternal.instances.{get,list}
- - iam.clouds.get
- - resource-manager.clouds.get
- - iam.serviceAccounts.issueCookie
- - iam.serviceAccounts.issueToken
- - iam.userAccounts.createAccessKey
- - iam.userAccounts.getAccessKey
- - iam.userAccounts.issueCookie
- - iam.userAccounts.issueToken
- - iam.userAccounts.presignUrl
- - iam.subjects.issueCookie
- - iam.subjects.issueToken
- - s3.objects.getObject
- - s3.objects.putObject
- - vpc.addresses.createExternal
- - vpc.addresses.createInternal
- - vpc.networks.create
- - vpc.networks.createDefaultSecurityGroup
- - vpc.networks.createRouteTable
- - vpc.networks.createSubnet
- - vpc.networks.use
- - vpc.subnets.create
- - vpc.subnets.assignExtraParams
-
- internal.tokenagent:
- groups: [iam]
- name: (Внутренняя) token agent
- visibility: internal
- description: Роль для инициализации хостов
- permissions:
- - iam.authorizedKeys.create
- - iam.serviceAccounts.createForHost
- - iam.tokenKeys.create
-
- internal.iamE2eTests:
- groups: [iam]
- name: (Внутренняя) IAM Private-API E2E Tests
- visibility: internal
- description: Роль для E2E тестов приватного API IAM
- permissions:
- - iam.serviceAccounts.issueCookie
- - iam.serviceAccounts.issueToken
- - iam.serviceAccounts.issueTokenForInstance
- - iam.userAccounts.issueCookie
- - iam.userAccounts.issueToken
- - iam.subjects.issueCookie
- - iam.subjects.issueToken
- - iam.resourceTypes.listAccessBindings
- - iam.resourceTypes.updateAccessBindings
- - iam.resourceTypes.listMemberships
- - iam.gizmo.listAccessBindings
- - iam.gizmo.updateAccessBindings
- - iam.root.listAccessBindings
- - iam.root.updateAccessBindings
-
- internal.iam.crossCloudBindings:
- groups: [iam]
- name: (Внутренняя) Редактор closs-cloud прав
- visibility: internal
- description: Системная роль для редактирования прав на ресурсы чужих облаков
- permissions:
- - iam.serviceAccounts.crossCloudBindings
- - iam.federatedUsers.crossCloudBindings
-
- internal.iam.accessBindings.viewer:
- groups: [iam]
- name: (Внутренняя) Чтение прав доступа
- visibility: internal
- description: Чтение прав доступа на ресурсах определённых типов и просмотр членства
- permissions:
- - iam.accessBindings.list
- - iam.resourceTypes.listMemberships
-
- internal.iam.accessBindings.admin:
- groups: [iam]
- name: (Внутренняя) Администрирование прав доступа
- visibility: internal
- description: Администрирование прав доступа на ресурсах определённых типов и просмотр членства
- includedRoles:
- - internal.iam.accessBindings.viewer
- permissions:
- - iam.accessBindings.update
-
- internal.iam.rootAccessBindingAdmin:
- groups: [iam]
- name: (Внутренняя) Администрирование прав доступа на root
- visibility: internal
- description: Администрирование прав доступа на корневой объект иерархии ресурсов
- permissions:
- - iam.root.listAccessBindings
- - iam.root.updateAccessBindings
-
- internal.iam.gizmoAccessBindingAdmin:
- groups: [iam]
- name: (Внутренняя) Администрирование прав доступа на gizmo
- visibility: internal
- description: Администрирование прав доступа на gizmo-ресурс
- permissions:
- - iam.gizmo.listAccessBindings
- - iam.gizmo.updateAccessBindings
-
- internal.iam.resourceTypes.admin:
- groups: [iam]
- name: (Внутренняя) Администрирование прав доступа на типы ресурсов
- visibility: internal
- description: Администрирование прав доступа на типы ресурсов
- permissions:
- - iam.resourceTypes.listAccessBindings
- - iam.resourceTypes.updateAccessBindings
-
- internal.iam.restrictionTypes.user:
- groups: [iam]
- name: (Внутренняя) Администрирование ограничениями
- visibility: internal
- description: Администрирование ограничения определённого типа на ресурсах
- permissions:
- - iam.restrictionTypes.use
-
- internal.iam.restrictionTypes.admin:
- groups: [iam]
- name: (Внутренняя) Администрирование прав доступа на типы ограничений
- visibility: internal
- description: Администрирование прав доступа на типы ограничений
- includedRoles:
- - internal.iam.restrictionTypes.user
- permissions:
- - iam.restrictionTypes.listAccessBindings
- - iam.restrictionTypes.updateAccessBindings
-
- internal.iam.quotas.admin:
- groups: [ iam ]
- name: (Внутренняя) Администрирование квот на ресурсы
- visibility: internal
- description: Администрирование лимитов квот на облака
- permissions:
- - iam.quotas.updateLimit
-
- internal.iam.agent:
- groups: [iam]
- name: (Внутренняя) IAM control plane
- visibility: internal
- description: Роль для инстансных SA IAM control plane
- permissions:
- - resource-manager.folders.get
- - organization-manager.organizations.listInvitations
-
- internal.iam.listResourceTypeMemberships:
- groups: [iam]
- name: (Внутренняя) Листинг членов ресурсов верхнего уровня
- visibility: internal
- description: Специальная роль для листинга членов ресурсов верхнего уровня
- permissions:
- - iam.resourceTypes.listMemberships
-
- internal.iam.sync:
- groups: [iam]
- name: Role for the service for syncing users and access rights with external systems
- visibility: internal
- description: ''
- permissions:
- - iam.operations.get
- - iam.operations.list
- - iam.accessBindings.list
- - iam.accessBindings.manageOwners
- - iam.accessBindings.update
- - iam.userAccounts.createManaged
- - iam.userOperations.list
- - iam.root.listAccessBindings
- - iam.root.updateAccessBindings
- - iam.serviceAccounts.listAccessBindings
- - iam.serviceAccounts.updateAccessBindings
- - iam.gizmo.listAccessBindings
- - iam.gizmo.updateAccessBindings
- - iam.userAccounts.getSshKey
- - iam.userAccounts.createSshKey
- - resource-manager.clouds.listAccessBindings
- - resource-manager.folders.listAccessBindings
-
- internal.iam.osLogins.admin:
- groups: [iam]
- name: (Внутренняя) Администрирование OS logins пользователей
- visibility: internal
- description: Просмотр и редактирование OS logins пользователей
- permissions:
- - iam.subjects.getOsLogin
- - iam.subjects.updateOsLogin
-
- internal.iam.sshKeys.admin:
- groups: [iam]
- name: (Внутренняя) Администрирование ssh keys пользователей и сервисных аккаунтов
- visibility: internal
- description: Просмотр и редактирование ssh keys пользователей и сервисных аккаунтов
- permissions:
- - iam.userAccounts.getSshKey
- - iam.userAccounts.createSshKey
- - iam.userAccounts.deleteSshKey
- - iam.serviceAccounts.getSshKey
- - iam.serviceAccounts.createSshKey
- - iam.serviceAccounts.deleteSshKey
-
- internal.iam.tokenCreatorFromUserJwt:
- groups: [iam]
- name: (Внутренняя) IAM роль для пользователей, разрешающая обменять JWT на IAM token
- visibility: internal
- description: Пользователей может обменять JWT на IAM token
- permissions:
- - iam.userAccounts.issueTokenFromJwt
-
- internal.iam.diagnostician:
- groups: [iam]
- name: (Внутренняя) Доступ к диагностическим ручкам IAM
- visibility: internal
- description: ''
- permissions:
- - iam.internal.diagnostics
-
- internal.iam.totpProfileCreator:
- groups: [iam]
- name: (Внутренняя) IAM роль для пользователей, разрешающая создавать TOTP профиль
- visibility: internal
- description: ''
- permissions:
- - iam.userAccounts.createTotpProfile
-
- internal.iam.userSettings.agent:
- groups: [iam]
- name: (Внутренняя) IAM роль для service accounts, разрешающая читать userSettings пользователей (locale,notifications)
- visibility: internal
- description: ''
- permissions:
- - iam.userAccounts.getSettings
-
- internal.iam.metaModelViewer:
- groups: [iam]
- name: (Внутренняя) Чтение метамодели авторизации
- visibility: internal
- description: ''
- permissions:
- - iam.permissionStages.get
- - iam.resourceTypes.get
- - iam.permissions.get
- - iam.roles.get
- - iam.oauthScopes.get
- - iam.oauthClients.get
- - iam.restrictionTypes.get
-
- internal.iam.metaModelEditor:
- groups: [iam]
- name: (Внутренняя) Изменение метамодели авторизации
- visibility: internal
- description: ''
- includedRoles:
- - internal.iam.metaModelViewer
- permissions:
- - iam.permissionStages.create
- - iam.restrictionTypes.{create,update}
- - iam.{resourceTypes,permissions,roles,oauthScopes,oauthClients}.create
- - iam.{resourceTypes,permissions,roles,oauthScopes,oauthClients}.update
-
- internal.iam.keyFingerprintSearcher:
- groups: [iam]
- name: (Внутренняя) Поиск ключей по fingerprint
- visibility: internal
- description: ''
- permissions:
- - iam.authorizedKeys.listByFingerprint
- - iam.accessKeys.listByFingerprint
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/root_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/root_service.proto
deleted file mode 100644
index d5777e8093..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/root_service.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PRTS";
-
-service RootService {
- rpc ListAccessBindings (ListRootAccessBindingsRequest) returns (access.ListAccessBindingsResponse);
-
- rpc UpdateAccessBindings (UpdateRootAccessBindingsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateRootAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message ListRootAccessBindingsRequest {
- int64 page_size = 1 [(value) = "<=1000"];
- string page_token = 2 [(length) = "<=100"];
-}
-
-message UpdateRootAccessBindingsRequest {
- repeated access.AccessBindingDelta access_binding_deltas = 1 [(size) = ">0"];
-}
-
-message UpdateRootAccessBindingsMetadata {
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate.proto
deleted file mode 100644
index 3efcf76b82..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate.proto
+++ /dev/null
@@ -1,39 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.saml;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml;saml";
-option java_outer_classname = "PSC";
-
-// A certificate.
-message Certificate {
-
- // ID of the certificate.
- string id = 1;
-
- // ID of the federation that the certificate belongs to.
- string federation_id = 2;
-
- // Name of the certificate. 3-63 characters long.
- string name = 3;
-
- // Description of the certificate. 0-256 characters long.
- string description = 4;
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 5;
-
- // Certificate data in PEM format.
- string data = 6;
-
- // SHA256-fingerprint of the certificate.
- string fingerprint = 7;
-
- // Time after which the certificate is not valid.
- google.protobuf.Timestamp not_after = 8;
-
- // Time before which the certificate is not valid.
- google.protobuf.Timestamp not_before = 9;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate_service.proto
deleted file mode 100644
index d1a5a1c733..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/certificate_service.proto
+++ /dev/null
@@ -1,110 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.saml;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/saml/certificate.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml;saml";
-option java_outer_classname = "PSCS";
-
-service CertificateService {
- rpc Get (GetCertificateRequest) returns (Certificate) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates/{certificate_id}" };
- }
-
- rpc List (ListCertificatesRequest) returns (ListCertificatesResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates" };
- }
-
- rpc Create (CreateCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/certificates" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateCertificateMetadata"
- response: "Certificate"
- };
- }
-
- // Updates the specified certificate.
- rpc Update (UpdateCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/saml/certificates/{certificate_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateCertificateMetadata"
- response: "Certificate"
- };
- }
-
- rpc Delete (DeleteCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/saml/certificates/{certificate_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteCertificateMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc ListOperations (ListCertificateOperationsRequest) returns (ListCertificateOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates/{certificate_id}/operations" };
- }
-}
-
-message GetCertificateRequest {
- string certificate_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListCertificatesRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
- string filter = 4 [(length) = "<=1000"];
-}
-
-message ListCertificatesResponse {
- repeated Certificate certificates = 1;
- string next_page_token = 2;
-}
-
-message CreateCertificateRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- string name = 2 [(pattern) = "([a-z]([-a-z0-9]{0,61}[a-z0-9])?)?"];
- string description = 3 [(length) = "<=256"];
- string data = 4 [(required) = true, (length) = "<=32000"];
-}
-
-message CreateCertificateMetadata {
- string certificate_id = 1;
-}
-
-message UpdateCertificateRequest {
- string certificate_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
- string name = 3 [(pattern) = "|[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 4 [(length) = "<=256"];
- string data = 5 [(length) = "<=32000"];
-}
-
-message UpdateCertificateMetadata {
- string certificate_id = 1;
-}
-
-message DeleteCertificateRequest {
- string certificate_id = 1 [(length) = "<=50"];
-}
-
-message DeleteCertificateMetadata {
- string certificate_id = 1;
-}
-
-message ListCertificateOperationsRequest {
- string certificate_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListCertificateOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation.proto
deleted file mode 100644
index 9d36085593..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation.proto
+++ /dev/null
@@ -1,92 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.saml;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml;saml";
-option java_outer_classname = "PSF";
-
-// A federation resource.
-// For more information, see [federations](/docs/iam/concepts/users/saml-federations).
-message Federation {
-
- // ID of the federation.
- string id = 1;
-
- // ID of the folder that the federation belongs to.
- string folder_id = 2;
-
- // Name of the federation.
- // The name is unique within the cloud. 3-63 characters long.
- string name = 3;
-
- // Description of the federation. 0-256 characters long.
- string description = 4;
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 5;
-
- // The 'IAM_COOKIE' cookie max age in seconds.
- // The value should be in range from 10 min to 12 hours.
- // The default value is 8 hours.
- google.protobuf.Duration cookie_max_age = 6;
-
- // Auto create accounts for new user on first login.
- bool auto_create_account_on_login = 7;
-
- // federation issuer (entity Id).
- string issuer = 8;
-
- // Single sign on endpoint binding type.
- BindingType sso_binding = 9;
-
- // Single sign on endpoint URL.
- string sso_url = 10;
-
- // Federation security settings.
- FederationSecuritySettings security_settings = 11;
-
- // Use case insensitive Name IDs.
- bool case_insensitive_name_ids = 12;
-}
-
-// SAML Binding is a mapping of a SAML protocol message onto standard messaging formats and/or communications protocols.
-enum BindingType {
-
- // Invalid
- BINDING_TYPE_UNSPECIFIED = 0;
-
- // HTTP POST binding.
- POST = 1;
-
- // HTTP redirect binding.
- REDIRECT = 2;
-
- // HTTP artifact binding.
- ARTIFACT = 3;
-
-}
-
-message FederationSecuritySettings {
- // Enable encrypted assertions.
- bool encrypted_assertions = 1;
-}
-
-message ServiceProviderCertificate {
- // ID of the federation that the certificate belongs to.
- string federation_id = 2;
-
- // Certificate data in PEM format.
- string data = 6;
-
- // SHA256-fingerprint of the certificate.
- string fingerprint = 7;
-
- // Time after which the certificate is not valid.
- google.protobuf.Timestamp not_after = 8;
-
- // Time before which the certificate is not valid.
- google.protobuf.Timestamp not_before = 9;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation_service.proto
deleted file mode 100644
index 0f9d1c9a59..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml/federation_service.proto
+++ /dev/null
@@ -1,171 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.saml;
-
-import "google/api/annotations.proto";
-import "google/protobuf/duration.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/user_account.proto";
-import "yandex/cloud/priv/iam/v1/saml/federation.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/saml;saml";
-option java_outer_classname = "PSFS";
-
-service FederationService {
- rpc Get (GetFederationRequest) returns (Federation) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}" };
- }
-
- rpc List (ListFederationsRequest) returns (ListFederationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations" };
- }
-
- rpc Create (CreateFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/federations" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateFederationMetadata"
- response: "Federation"
- };
- }
-
- rpc Update (UpdateFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/saml/federations/{federation_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateFederationMetadata"
- response: "Federation"
- };
- }
-
- rpc Delete (DeleteFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/saml/federations/{federation_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteFederationMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc AddUserAccounts (AddFederatedUserAccountsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/federations/{federation_id}:addUserAccounts" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "AddFederatedUserAccountsMetadata"
- response: "AddFederatedUserAccountsResponse"
- };
- }
-
- rpc ListUserAccounts (ListFederatedUserAccountsRequest) returns (ListFederatedUserAccountsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}:listUserAccounts" };
- }
-
- rpc ListOperations (ListFederationOperationsRequest) returns (ListFederationOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}/operations" };
- }
-
- rpc GetServiceProviderCertificate (GetServiceProviderCertificateRequest) returns (ServiceProviderCertificate);
-}
-
-message GetFederationRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message GetServiceProviderCertificateRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListFederationsRequest {
- oneof scope {
- option (exactly_one) = true;
- string cloud_id = 1 [(length) = "<=50"];
- string folder_id = 2 [(length) = "<=50"];
- }
- int64 page_size = 3 [(value) = "0-1000"];
- string page_token = 4 [(length) = "<=100"];
- string filter = 5 [(length) = "<=1000"];
-}
-
-message ListFederationsResponse {
- repeated Federation federations = 1;
- string next_page_token = 2;
-}
-
-message CreateFederationRequest {
- string id = 11 [(length) = "<=50"];
- string folder_id = 1 [(required) = true, (length) = "<=50"];
- string name = 2 [(required) = true, (pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 3 [(length) = "<=256"];
- google.protobuf.Duration cookie_max_age = 4 [(value) = "10m-12h"];
- bool auto_create_account_on_login = 5;
- string issuer = 6 [(required) = true, (length) = "<=8000"];
- BindingType sso_binding = 7 [(required) = true];
- string sso_url = 8 [(required) = true, (length) = "<=8000"];
- FederationSecuritySettings security_settings = 9;
- bool case_insensitive_name_ids = 10;
-}
-
-message CreateFederationMetadata {
- string federation_id = 1;
-}
-
-message UpdateFederationRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
-
- string name = 3 [(pattern) = "|[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 4 [(length) = "<=256"];
- google.protobuf.Duration cookie_max_age = 5 [(value) = "10m-12h"];
- bool auto_create_account_on_login = 6;
- string issuer = 7 [(length) = "<=8000"];
- BindingType sso_binding = 8;
- string sso_url = 9 [(length) = "<=8000"];
- FederationSecuritySettings security_settings = 10;
- bool case_insensitive_name_ids = 11;
-}
-
-message UpdateFederationMetadata {
- string federation_id = 1;
-}
-
-message DeleteFederationRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteFederationMetadata {
- string federation_id = 1;
-}
-
-message AddFederatedUserAccountsRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- repeated string name_ids = 2 [(size) = "1-1000", (length) = "1-1000"];
-}
-
-message AddFederatedUserAccountsMetadata {
- string federation_id = 1;
-}
-
-message AddFederatedUserAccountsResponse {
- repeated UserAccount user_accounts = 1;
-}
-
-message ListFederatedUserAccountsRequest {
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListFederatedUserAccountsResponse {
- repeated UserAccount user_accounts = 1;
- string next_page_token = 2;
-}
-
-message ListFederationOperationsRequest {
- string federation_id = 1 [(length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListFederationOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/scopes.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/scopes.yaml
deleted file mode 100644
index b1f0f1e0d5..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/scopes.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-scopes:
-
- yc.iam.accessBindings.manage:
- service: iam
- name: 'управление биндингами'
- description: ''
- visibility: internal
- permissions:
- - iam.accessBindings.list
- - iam.accessBinding.delete # TODO remove ?
- - iam.accessBindings.set
- - iam.accessBindings.update
-
- yc.iam.clouds.manage:
- service: iam
- name: 'удалять облако и редактировать его настройки'
- description: ''
- visibility: internal
- permissions:
- - iam.clouds.delete
- - iam.clouds.setUserListingSetting
-
- yc.iam.federations.manage:
- service: iam
- name: 'управление федерациями'
- description: ''
- visibility: internal
- permissions:
- - iam.federations.createCertificate
- - iam.federations.createUser
- - iam.federations.deleteCertificate
- - iam.federations.deleteUser
- - iam.federations.getCertificate
- - iam.federations.listCertificate
- - iam.federations.updateCertificate
- - iam.federations.{create,delete,update}
- - iam.federations.{get,list}
-
- yc.iam.serviceAccounts.manage:
- service: iam
- name: 'управление сервисными аккаунтами'
- description: ''
- visibility: internal
- permissions:
- - iam.tokenKeys.{create,delete,update}
- - iam.authorizedKeys.{create,delete,update}
- - iam.serviceAccounts.{create,update,delete}AccessKey
- - iam.serviceAccounts.{create,update,delete}ApiKey
- - iam.serviceAccounts.{create,delete}SshKey
- - iam.serviceAccounts.{create,delete,update}
-
- yc.iam.serviceAccounts.apiKey.get:
- service: iam
- name: 'получение информации об API-ключах'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.{getApiKey,listApiKeys}
-
- yc.iam.serviceAccounts.accessKey.get:
- service: iam
- name: 'получение информации об авторизованных ключах'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.{getAccessKey,listAccessKeys}
-
- yc.iam.serviceAccounts.sshKey.get:
- service: iam
- name: 'получение информации об SSH-ключах'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.getSshKey
-
- yc.iam.serviceAccounts.get:
- service: iam
- name: 'получение информации о сервисных аккаунтах'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.{get,list}
-
- yc.iam.serviceAccounts.use:
- service: iam
- name: 'использование сервисного аккаунта'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.use
-
- yc.iam.serviceAccounts.issueToken:
- service: iam
- name: 'получения токена сервисного-аккаунта'
- description: ''
- visibility: internal
- permissions:
- - iam.serviceAccounts.issueToken
-
- yc.iam.authorizedKeys.get:
- service: iam
- name: 'чтение авторизованных ключей'
- description: ''
- visibility: internal
- permissions:
- - iam.authorizedKeys.get
- - iam.tokenKeys.get
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account.proto
deleted file mode 100644
index 8d39273180..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account.proto
+++ /dev/null
@@ -1,20 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PSA";
-
-message ServiceAccount {
-
- string id = 1;
- string folder_id = 2;
-
- google.protobuf.Timestamp created_at = 3;
-
- string name = 4;
- string description = 5;
- map<string, string> labels = 6;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account_service.proto
deleted file mode 100644
index def7bc159e..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/service_account_service.proto
+++ /dev/null
@@ -1,146 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/iam/v1/token/iam_token.proto";
-import "yandex/cloud/priv/iam/v1/service_account.proto";
-import "yandex/cloud/priv/access/access.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PSAS";
-
-service ServiceAccountService {
- rpc Get (GetServiceAccountRequest) returns (ServiceAccount) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{service_account_id}" };
- }
-
- rpc List (ListServiceAccountsRequest) returns (ListServiceAccountsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts" };
- }
-
- rpc Create (CreateServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateServiceAccountMetadata"
- response: "ServiceAccount"
- };
- }
-
- rpc Update (UpdateServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/serviceAccounts/{service_account_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateServiceAccountMetadata"
- response: "ServiceAccount"
- };
- }
-
- rpc Delete (DeleteServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/serviceAccounts/{service_account_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteServiceAccountMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- //access
-
- rpc ListAccessBindings (access.ListAccessBindingsRequest) returns (access.ListAccessBindingsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{resource_id}:listAccessBindings" };
- }
-
- rpc SetAccessBindings (access.SetAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts/{resource_id}:setAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "access.SetAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc UpdateAccessBindings (access.UpdateAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts/{resource_id}:updateAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "access.UpdateAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- option (yandex.cloud.api.tools.method).lint_skip.http_verb = true;
- }
-
- rpc ListOperations (ListServiceAccountOperationsRequest) returns (ListServiceAccountOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{service_account_id}/operations" };
- }
-
- rpc IssueToken (IssueTokenRequest) returns (IamToken) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts/{service_account_id}:issueToken" body: "*" };
- }
-}
-
-message GetServiceAccountRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListServiceAccountsRequest {
- string folder_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
- string filter = 4 [(length) = "<=1000"];
-}
-
-message ListServiceAccountsResponse {
- repeated ServiceAccount service_accounts = 1;
- string next_page_token = 2;
-}
-
-message CreateServiceAccountRequest {
- string folder_id = 1 [(required) = true, (length) = "<=50"];
- string name = 2 [(required) = true, (pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 3 [(length) = "<=256"];
- string id = 4 [(length) = "<=50"];
- map<string, string> labels = 5 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-}
-
-message CreateServiceAccountMetadata {
- string service_account_id = 1;
-}
-
-message UpdateServiceAccountRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
- google.protobuf.FieldMask update_mask = 2;
-
- string name = 3 [(pattern) = "|[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
- string description = 4 [(length) = "<=256"];
- map<string, string> labels = 5 [(priv.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-}
-
-message UpdateServiceAccountMetadata {
- string service_account_id = 1;
-}
-
-message DeleteServiceAccountRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteServiceAccountMetadata {
- string service_account_id = 1;
-}
-
-message ListServiceAccountOperationsRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListServiceAccountOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
-
-message IssueTokenRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
- string instance_id = 2 [(length) = "<=50"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key.proto
deleted file mode 100644
index e2888739b0..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PSK";
-
-message SshKey {
- string id = 1;
- string data = 2;
- string fingerprint = 3;
- google.protobuf.Timestamp created_at = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key_service.proto
deleted file mode 100644
index fcfaf8d18a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ssh_key_service.proto
+++ /dev/null
@@ -1,86 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/ssh_key.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PSKS";
-
-service SshKeyService {
- rpc Get (GetSshKeyRequest) returns (SshKey);
-
- rpc List (ListSshKeysRequest) returns (ListSshKeysResponse);
-
- rpc Create (CreateSshKeyRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "CreateSshKeyMetadata"
- response: "SshKey"
- };
- }
-
- rpc Delete (DeleteSshKeyRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteSshKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc ListOperations (ListSshKeyOperationsRequest) returns (ListSshKeyOperationsResponse);
-}
-
-message GetSshKeyRequest {
- string ssh_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListSshKeysRequest {
- oneof subject {
- string service_account_id = 1;
- string user_account_id = 2;
- string federated_user_id = 3;
- }
- int64 page_size = 4 [(value) = "0-1000"];
- string page_token = 5 [(length) = "<=100"];
-}
-
-message ListSshKeysResponse {
- repeated SshKey ssh_keys = 1;
- string next_page_token = 2;
-}
-
-message CreateSshKeyRequest {
- string ssh_key_id = 1 [(length) = "<=50"];
- oneof subject {
- string service_account_id = 2;
- string user_account_id = 3;
- string federated_user_id = 4;
- }
- string ssh_key_data = 5 [(required) = true, (length) = "<=20000"];
-}
-
-message CreateSshKeyMetadata {
- string ssh_key_id = 1;
- string subject_id = 2;
-}
-
-message DeleteSshKeyRequest {
- string ssh_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteSshKeyMetadata {
- string ssh_key_id = 1;
-}
-
-message ListSshKeyOperationsRequest {
- string ssh_key_id = 1 [(required) = true, (length) = "<=50"];
- int64 page_size = 2 [(value) = "0-1000"];
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListSshKeyOperationsResponse {
- repeated operation.Operation operations = 1;
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/subject_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/subject_service.proto
deleted file mode 100644
index 7932e4b8e2..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/subject_service.proto
+++ /dev/null
@@ -1,81 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PSS";
-
-service SubjectService {
- rpc GetOrCreate (GetOrCreateSubjectRequest) returns (GetOrCreateSubjectResponse);
-
- rpc Merge (MergeSubjectsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "MergeSubjectsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc Delete (DeleteSubjectRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteSubjectMetadata"
- response: "google.protobuf.Empty"
- };
- }
-}
-
-message GetOrCreateSubjectRequest {
- // Optional IAM Subject unique identifier (well-known-id).
- string subject_id = 1 [(required) = false, (length) = "<=50"];
- // IAM Subject type. May be one of:
- // * serviceAccount (unsupported yet)
- // * federatedUser (unsupported yet)
- // * invitee
- string subject_type = 2 [(required) = true, (length) = "<=256"];
- // The Subject external (IdP-specific) identifier.
- string external_id = 3 [(required) = true, (length) = "3-256"];
- // Json-string representation of map<string, object> IAM Subject attributes.
- string attributes = 4 [(length) = "<=262144"];
- // Json-string representation of map<string, object> IAM Subject settings.
- string settings = 5 [(length) = "<=262144"];
-}
-
-message GetOrCreateSubjectResponse {
- // IAM Subject unique identifier.
- string id = 1;
- // IAM Subject type.
- string subject_type = 2;
- // The Subject external (IdP-specific) identifier.
- string external_id = 3;
- // Json-string representation of map<string, object> IAM Subject attributes.
- string attributes = 4;
- // Json-string representation of map<string, object> IAM Subject settings.
- string settings = 5;
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 6;
-}
-
-message MergeSubjectsRequest {
- // Source subject.
- // Attention!!! Source subject will be deleted at the end of the merge operation.
- string source_subject_id = 1 [(required) = true];
- // Destination subject.
- string destination_subject_id = 2 [(required) = true];
-}
-
-message MergeSubjectsMetadata {
- string source_subject_id = 1;
- string destination_subject_id = 2;
-}
-
-message DeleteSubjectRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteSubjectMetadata {
- string subject_id = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/CMakeLists.txt
deleted file mode 100644
index 18e54bc56f..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(iam-v1-token)
-set_property(TARGET iam-v1-token PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET iam-v1-token PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(iam-v1-token PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(iam-v1-token PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(iam-v1-token PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(iam-v1-token PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/iam_token.proto
-)
-target_proto_addincls(iam-v1-token
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(iam-v1-token
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(iam-v1-token
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/iam_token.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/iam_token.proto
deleted file mode 100644
index 8c44532a2b..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token/iam_token.proto
+++ /dev/null
@@ -1,14 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token;iam";
-option java_outer_classname = "PIT";
-
-message IamToken {
- string iam_token = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
- google.protobuf.Timestamp expires_at = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token_agent.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token_agent.proto
deleted file mode 100644
index b77df757ef..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/token_agent.proto
+++ /dev/null
@@ -1,26 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PTAS";
-
-// The IAM token agent definition.
-service TokenAgent {
- rpc GetToken(GetTokenRequest) returns (GetTokenResponse) {}
-}
-
-message GetTokenRequest {
- // An additional service qualifier for obtaining different tokens
- // for different services running on behalf of the same user.
- string tag = 1;
-}
-
-// The response message containing the token
-message GetTokenResponse {
- string iam_token = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
- google.protobuf.Timestamp expires_at = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/agreement_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/agreement_service.proto
deleted file mode 100644
index 8113c3c03f..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/agreement_service.proto
+++ /dev/null
@@ -1,36 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/api/tools/options.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTAGS";
-
-// TODO console
-service AgreementService {
- rpc AcceptAgreements (AcceptAgreementsRequest) returns (AcceptAgreementsResponse) {
- option (google.api.http) = { post: "/iam/v1/acceptAgreements" body: "*" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc AcceptAgreementsOauth (AcceptAgreementsRequest) returns (AcceptAgreementsResponse) {
- option (google.api.http) = { post: "/iam/v1/acceptAgreementsOauth" body: "*" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc AcceptAgreementsLogin (AcceptAgreementsRequest) returns (AcceptAgreementsResponse) {
- option (google.api.http) = { post: "/iam/v1/acceptAgreementsLogin" body: "*" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-}
-
-message AcceptAgreementsRequest {
- string eula = 1;
- string privacy_policy = 2;
-}
-
-message AcceptAgreementsResponse {
- string agreements = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/auth_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/auth_service.proto
deleted file mode 100644
index b8f8b4c502..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/auth_service.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTAS";
-
-service AuthService {
- // TODO legacy v2
- rpc SessionId (SessionIdRequest) returns (SessionIdResponse) {
- option (google.api.http) = { post: "/iam/v1/auth/session_id" body: "*" };
- }
-}
-
-message SessionIdRequest {
- string organization_id = 1;
-}
-
-message SessionIdResponse {
- string token = 1 [(sensitive) = true];
- string secret_key = 2 [(sensitive) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/health_check_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/health_check_service.proto
deleted file mode 100644
index 6ef824087a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/health_check_service.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/api/tools/options.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTHCS";
-
-service HealthCheckService {
- rpc GetHealth (HealthCheckRequest) returns (HealthCheckResponse) {
- option (google.api.http) = { get: "/iam/v1/health" };
- }
-}
-
-message HealthCheckRequest {
-}
-
-message HealthCheckResponse {
- HealthStatus health = 1;
-
- enum HealthStatus {
- option (cloud.api.tools.enumeration).lint_skip.unspecified_value = true;
- PASSED = 0;
- FAILED = 1;
- UNKNOWN = 2;
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/passport_federation_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/passport_federation_service.proto
deleted file mode 100644
index 3bca566fcf..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/passport_federation_service.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/iam/v1/user_account.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTPFS";
-
-service PassportFederationService {
-
- rpc AddUserAccounts (AddPassportUserAccountsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "AddUserAccountsMetadata"
- response: "AddUserAccountsResponse"
- };
- }
-
-}
-
-message AddPassportUserAccountsRequest {
- oneof ids {
- option (exactly_one) = true;
- Logins logins = 1;
- Uids uids = 2;
- }
-}
-
-message Uids {
- repeated int64 values = 1 [(size) = "<=1000", (value) = ">0"];
-}
-
-message Logins {
- repeated string values = 1 [(size) = "<=1000", (length) = "2-63"];
-}
-
-message AddUserAccountsMetadata {
-}
-
-message AddUserAccountsResponse {
- repeated UserAccount user_accounts = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/policy_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/policy_service.proto
deleted file mode 100644
index 7a20166b50..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/policy_service.proto
+++ /dev/null
@@ -1,79 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTPS";
-
-service PolicyService {
- rpc List (ListPoliciesRequest) returns (ListPoliciesResponse) {
- option (google.api.http) = { get: "/iam/v1/cloud/{cloud_id}/policies" };
- }
-
- rpc ListCompat (ListPoliciesCompatRequest) returns (ListPoliciesResponse) {
- option (google.api.http) = { get: "/iam/v1/org/{org_id}/policies" };
- }
-
- rpc Set (SetPolicyRequest) returns (SetPolicyResponse) {
- option (google.api.http) = { post: "/iam/v1/cloud/{cloud_id}/policies" body: "*" };
- }
-
- rpc SetCompat (SetPolicyCompatRequest) returns (SetPolicyResponse) {
- option (google.api.http) = { post: "/iam/v1/org/{org_id}/policies" body: "*" };
- }
-
- rpc Delete (DeletePolicyRequest) returns (DeletePolicyResponse) {
- option (google.api.http) = { delete: "/iam/v1/cloud/{cloud_id}/policy/{assignment_id}" };
- }
-
- rpc DeleteCompat (DeletePolicyCompatRequest) returns (DeletePolicyResponse) {
- option (google.api.http) = { delete: "/iam/v1/cloud/{org_id}/policy/{assignment_id}" };
- }
-}
-
-message ListPoliciesRequest {
- string cloud_id = 1;
-}
-
-message ListPoliciesCompatRequest {
- string org_id = 1;
-}
-
-message Assignment {
- string id = 1;
-}
-
-message ListPoliciesResponse {
- repeated Assignment result = 1;
-}
-
-message SetPolicyRequest {
- string cloud_id = 1;
- string policy_id = 2;
-}
-
-message SetPolicyCompatRequest {
- string org_id = 1;
- string policy_id = 2;
-}
-
-message SetPolicyResponse {
- string status = 1;
- string id = 2;
-}
-
-message DeletePolicyRequest {
- string cloud_id = 1;
- string assignment_id = 2;
-}
-
-message DeletePolicyCompatRequest {
- string org_id = 1;
- string assignment_id = 2;
-}
-
-message DeletePolicyResponse {
- string status = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/roles_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/roles_service.proto
deleted file mode 100644
index 8c155e80b5..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/roles_service.proto
+++ /dev/null
@@ -1,28 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTRS";
-
-service RoleService {
- rpc List (ListRolesRequest) returns (ListRolesResponse) {
- option (google.api.http) = { get: "/iam/v1/role" };
- }
-}
-
-message ListRolesRequest {
- string language = 1;
-}
-
-message ListRolesResponse {
- repeated Role result = 1;
-
- message Role {
- string slug = 1;
- string name = 2;
- string desc = 3;
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/service_account_access_key_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/service_account_access_key_service.proto
deleted file mode 100644
index 2ab344f4a0..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/service_account_access_key_service.proto
+++ /dev/null
@@ -1,54 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-import "google/protobuf/empty.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTSAAKS";
-
-service ServiceAccountAccessKeyService {
- rpc PresignUrl (PresignUrlRequest) returns (PresignUrlResponse) {
- option (google.api.http) = { post: "/iam/v1/presignURL" body: "*" };
- }
-
- rpc PresignUrls (PresignUrlsRequest) returns (PresignUrlsResponse) {
- option (google.api.http) = { post: "/iam/v1/presignURLs" body: "*" };
- }
-
- rpc GetCurrentPresignKey (google.protobuf.Empty) returns (PresignKey) {
- option (google.api.http) = { get: "/iam/v1/presignURL:getCurrentKey" };
- }
-}
-
-message PresignUrlRequest {
- string key_id = 1; // TODO not cloud ID
- string string_to_sign = 2;
- string version = 3; // TODO enum
- string region = 4;
- string service = 5;
-}
-
-message PresignUrlResponse {
- string result = 1;
- string signature = 2;
- string key_id = 3;
-}
-
-message PresignUrlsRequest {
- string key_id = 1; // TODO not cloud ID
- repeated string strings_to_sign = 2;
- string version = 3; // TODO enum
- string region = 4;
- string service = 5;
-}
-
-message PresignUrlsResponse {
- string key_id = 1;
- repeated string signatures = 2;
-}
-
-message PresignKey {
- string key_id = 1;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/user_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/user_service.proto
deleted file mode 100644
index 016f3a23e7..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional/user_service.proto
+++ /dev/null
@@ -1,90 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.transitional;
-
-import "google/api/annotations.proto";
-import "google/protobuf/empty.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/transitional;iam";
-option java_outer_classname = "PTUS";
-
-service UserService {
- rpc GetMetadata (GetUserMetadataRequest) returns (GetUserMetadataResponse) {
- option (google.api.http) = { get: "/iam/v1/org/{org_id}/user/{user_id}/metadata" };
- }
-
- rpc SetMetadata (SetUserMetadataRequest) returns (SetUserMetadataResponse) {
- option (google.api.http) = { post: "/iam/v1/org/{org_id}/user/{user_id}/metadata" body: "*" };
- }
-
- rpc GetPassportUser (GetPassportUserRequest) returns (GetPassportUserResponse) {
- option (google.api.http) = { get: "/iam/v1/passportUid" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc ListUsers (ListUsersRequest) returns (ListUsersResponse) {
- option (google.api.http) = { get: "/iam/v1/cloud/{cloud_id}/users" };
- option (yandex.cloud.api.tools.method).lint_skip.contains_resource_name = true;
- }
-
- rpc GetCurrentSubject (google.protobuf.Empty) returns (GetCurrentSubjectResponse) {
- option (google.api.http) = { get: "/iam/v1/whoami" };
- }
-}
-
-message GetUserMetadataRequest {
- string org_id = 1;
- string user_id = 2;
-}
-
-message GetUserMetadataResponse {
- string metadata = 1 [(sensitive) = true, (sensitive_type) = SENSITIVE_CRC];
-}
-
-message SetUserMetadataRequest {
- string org_id = 1;
- string user_id = 2;
- string metadata = 3[(sensitive) = true, (sensitive_type) = SENSITIVE_CRC];
-}
-
-message SetUserMetadataResponse {
- string status = 1;
-}
-
-message GetPassportUserRequest {
- string subject_id = 1;
-}
-
-message GetPassportUserResponse {
- string passport_uid = 1;
-}
-
-message ListUsersRequest {
- string cloud_id = 1;
-}
-
-message ListUsersResponse {
- repeated User users = 1;
-
- message User {
- string id = 1;
- string login = 2;
- string first_name = 3;
- string last_name = 4;
- string avatar = 5;
- }
-}
-
-message GetCurrentSubjectResponse {
- string id = 1;
- string cloud_id = 2;
- google.protobuf.Timestamp created_at = 3;
- string subject_type = 4;
- string login = 5;
- string email = 6;
- string reference = 7;
- int64 deleted = 8;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/CMakeLists.txt
deleted file mode 100644
index 836769b275..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(iam-v1-ts)
-set_property(TARGET iam-v1-ts PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET iam-v1-ts PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(iam-v1-ts PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(iam-v1-ts PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(iam-v1-ts PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(iam-v1-ts PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto
-)
-target_proto_addincls(iam-v1-ts
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(iam-v1-ts
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(iam-v1-ts
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto
deleted file mode 100644
index 0081c4b5af..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1.ts;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/ts;iam";
-option java_outer_classname = "PITS";
-
-message Subject {
- oneof type {
- option (exactly_one) = true;
-
- UserAccount user_account = 1;
- ServiceAccount service_account = 2;
- }
-
- message UserAccount {
- string id = 1 [(required) = true, (length) = "<=50"];
- }
-
- message ServiceAccount {
- string id = 1 [(required) = true, (length) = "<=50"];
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account.proto
deleted file mode 100644
index bdc48a2ea0..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account.proto
+++ /dev/null
@@ -1,46 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PUA";
-
-message UserAccount {
- string id = 1;
- oneof user_account {
- option (exactly_one) = true;
- YandexPassportUserAccount yandex_passport_user_account = 2;
- SamlUserAccount saml_user_account = 3;
- OAuthUserAccount oauth_user_account = 4;
- }
-}
-
-message YandexPassportUserAccount {
- string login = 1;
- string default_email = 2;
-}
-
-// A SAML federated user.
-// For more information, see [federations](/docs/iam/concepts/users/saml-federations).
-message SamlUserAccount {
- message Attribute {
- repeated string value = 1;
- }
- // ID of the federation that the user belongs to.
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- // Name Id of the SAML federated user.
- // The name is unique within the federation. 1-256 characters long.
- string name_id = 2 [(required) = true, (length) = "1-256"];
- // Additional attributes of the SAML federated user.
- map <string, Attribute> attributes = 3;
-}
-
-// A OAuth federated user.
-message OAuthUserAccount {
- // ID of the federation that the user belongs to.
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- // Federated user claims.
- map <string, string> claims = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account_service.proto
deleted file mode 100644
index 671ae5ca91..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/user_account_service.proto
+++ /dev/null
@@ -1,106 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/iam/v1/user_account.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PUAS";
-
-service UserAccountService {
- rpc Get (GetUserAccountRequest) returns (UserAccount) {
- option (google.api.http) = { get: "/iam/v1/userAccounts/{user_account_id}" };
- }
-
- rpc Delete (DeleteUserAccountRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteUserAccountMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- rpc GetSettings (GetSettingsRequest) returns (UserSettings);
-
- rpc UpdateSettings (UpdateSettingsRequest) returns (UserSettings);
-
- rpc PresignUrl(PresignUrlRequest) returns (PresignUrlResponse);
-}
-
-message GetUserAccountRequest {
- string user_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteUserAccountRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteUserAccountMetadata {
- string subject_id = 1;
-}
-
-message GetSettingsRequest {
- // Empty list means full settings.
- repeated string response_json_path = 1 [(size) = "<=100", (length) = "1-1000"];
- // Optional - get specified subject user settings. By default equals to authenticated subject.
- string subject_id = 2 [(length) = "<=50"];
-}
-
-message UserSettings {
- // JSON-serialized user-settings.
- string json = 1;
-}
-
-message UpdateSettingsRequest {
- // Empty list means full settings.
- repeated string response_json_path = 1 [(size) = "<=100", (length) = "1-1000"];
- // Serialized JSON Patch (https://tools.ietf.org/html/rfc6902).
- string json_patch = 2 [(length) = "<=10000"];
-}
-
-message PresignUrlRequest {
- string subject_id = 1 [(required) = true, (length) = "<=50"];
- // The formatted string to sign, see https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
- repeated string strings_to_sign = 2 [(size) = "1-100", (length) = "1-1024", (unique) = true];
-
- oneof parameters {
- option (exactly_one) = true;
-
- Version2Parameters v2_parameters = 4;
- Version4Parameters v4_parameters = 5;
- }
-
- // https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
- message Version2Parameters {
- SignatureMethod signature_method = 1 [(required) = true];
-
- enum SignatureMethod {
- SIGNATURE_METHOD_UNSPECIFIED = 0;
- HMAC_SHA1 = 1;
- HMAC_SHA256 = 2;
- }
- }
-
- // https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
- message Version4Parameters {
- google.protobuf.Timestamp signed_at = 1 [(required) = true];
- string service = 2 [(required) = true, (length) = "<=64"];
- string region = 3 [(required) = true, (length) = "<=32"];
- }
-
-}
-
-message PresignUrlResponse {
- string access_key_id = 1;
- repeated SignedString signed_strings = 2;
-}
-
-message SignedString {
- string string_to_sign = 1;
- string signature = 2 [(sensitive) = true, (sensitive_type) = SENSITIVE_CRC];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto
deleted file mode 100644
index 6b493101c1..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_cookie.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PYPC";
-
-message YandexPassportCookies {
- // Cookies Session_id and sessionid2
- // https://doc.yandex-team.ru/blackbox/reference/MethodSessionID.html
-
- // Also blackbox verify client's IP to belong to internal networks.
- // Internal services set different cookies depending on which network the user's IP - external or internal.
- //
- // Use X-Forwarded-For to send user's IP which will be sent to blackbox.
- //
- // IP address need specify in standard IPv4 format (for example, 194.84.46.241)
- // or IPv6 (for example, 2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d).
- // If IP address in the wrong format, returns an error message.
-
- // Value of the cookie being checked Session_id.
- string session_id = 1 [(required) = true, (length) = "<=4093", (sensitive) = true];
-
- // The value of a cookie sessionid2.
- string ssl_session_id = 2 [(length) = "<=4093", (sensitive) = true];
-
- // Host address, for example "yandex.ua" or "volozh.ya.ru".
- string host = 3 [(required) = true, (length) = "<=253"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_user_account_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_user_account_service.proto
deleted file mode 100644
index 733dfd2660..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/yandex_passport_user_account_service.proto
+++ /dev/null
@@ -1,52 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.iam.v1;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/priv/operation/operation.proto";
-import "yandex/cloud/priv/iam/v1/user_account.proto";
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1;iam";
-option java_outer_classname = "PYPUAS";
-
-service YandexPassportUserAccountService {
- rpc GetByLogin (GetUserAccountByLoginRequest) returns (UserAccount) {
- option (google.api.http) = { get: "/iam/v1/yandexPassportUserAccounts:byLogin" };
- }
-
- // create subject_id for Yandex.Passport user identifier.
- rpc AddUserAccounts (AddPassportUserAccountsRequest) returns (operation.Operation) {
- option (yandex.cloud.api.operation) = {
- metadata: "AddUserAccountsMetadata"
- response: "AddUserAccountsResponse"
- };
- }
-}
-
-message GetUserAccountByLoginRequest {
- string login = 1 [(required) = true];
-}
-
-message AddUserAccountsMetadata {
-}
-
-message AddUserAccountsResponse {
- repeated UserAccount valid_users = 1;
- repeated PassportUser invalid_users = 2;
-}
-
-
-message AddPassportUserAccountsRequest {
- repeated PassportUser passport_users = 1 [(size) = "1-100"];
-}
-
-message PassportUser {
- oneof identity {
- option (exactly_one) = true;
- // Yandex.Passport login or Yandex.Passport email
- string login = 1 [(length) = "2-63"];
- int64 uid = 2 [(value) = ">0"];
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/CMakeLists.txt
deleted file mode 100644
index 7efbdab52f..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-priv-oauth)
-set_property(TARGET cloud-priv-oauth PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-priv-oauth PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-oauth PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-oauth PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(cloud-priv-oauth PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-priv-oauth PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/claims.proto
-)
-target_proto_addincls(cloud-priv-oauth
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-priv-oauth
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(cloud-priv-oauth
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/claims.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/claims.proto
deleted file mode 100644
index 8f3d42426a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/claims.proto
+++ /dev/null
@@ -1,90 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.oauth;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/oauth;oauth";
-option java_outer_classname = "Claims";
-
-import "yandex/cloud/priv/validation.proto";
-
-// claims representation, see https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims for details
-message SubjectClaims {
- // Subject - Identifier for the End-User at the Issuer.
- string sub = 1 [(required) = true, (length) = "<=50"];
- // End-User's full name in displayable form including all name parts, possibly including titles and suffixes, ordered according to the End-User's locale and preferences.
- string name = 2;
- // Given name(s) or first name(s) of the End-User. Note that in some cultures, people can have multiple given names; all can be present, with the names being separated by space characters.
- string given_name = 3;
- // Surname(s) or last name(s) of the End-User. Note that in some cultures, people can have multiple family names or no family name; all can be present, with the names being separated by space characters.
- string family_name = 4;
- // Shorthand name by which the End-User wishes to be referred to at the RP, such as janedoe or j.doe.
- // This value MAY be any valid JSON string including special characters such as @, /, or whitespace. The RP MUST NOT rely upon this value being unique, as discussed in Section 5.7.
- string preferred_username = 7;
- // URL of the End-User's profile picture. This URL MUST refer to an image file (for example, a PNG, JPEG, or GIF image file),
- // rather than to a Web page containing an image. Note that this URL SHOULD specifically reference a profile photo of the End-User suitable for displaying when describing the End-User, rather than an arbitrary photo taken by the End-User.
- string picture = 9;
- // End-User's preferred e-mail address. Its value MUST conform to the RFC 5322 [RFC5322] addr-spec syntax.
- // The RP MUST NOT rely upon this value being unique, as discussed in Section 5.7.
- string email = 11;
- // String from zoneinfo [zoneinfo] time zone database representing the End-User's time zone. For example, Europe/Paris or America/Los_Angeles.
- string zoneinfo = 15;
- // End-User's locale, represented as a BCP47 [RFC5646] language tag. This is typically an ISO 639-1 Alpha-2 [ISO639-1] language code in lowercase and an ISO 3166-1 Alpha-2 [ISO3166-1] country code in uppercase, separated by a dash.
- // For example, en-US or fr-CA. As a compatibility note, some implementations have used an underscore as the separator rather than a dash, for example, en_US; Relying Parties MAY choose to accept this locale syntax as well.
- string locale = 16;
- // End-User's preferred telephone number. E.164 [E.164] is RECOMMENDED as the format of this Claim, for example, +1 (425) 555-1212 or +56 (2) 687 2400.
- // If the phone number contains an extension, it is RECOMMENDED that the extension be represented using the RFC 3966 [RFC3966] extension syntax, for example, +1 (604) 555-1234;ext=5678.
- string phone_number = 17;
-
-
- // Subject type.
- SubjectType sub_type = 99;
- // non-empty only for federated users
- Federation federation = 100;
- // An image content of the user. Typically a space-efficient format like JPEG or GIF encoded in base64-string.
- // The maximum size is limited to 100 Kib.
- string picture_data = 101;
- YandexClaims yandex_claims = 200;
-}
-
-// minimalistic analog of yandex.cloud.priv.iam.v1.saml.Federation
-message Federation {
- // ID of the federation.
- string id = 1 [(required) = true, (length) = "<=50"];
- // Name of the federation. The name is unique within the cloud or organization
- string name = 3;
- // Whether the federation is global or not.
- bool is_global = 5;
-}
-
-message YandexClaims {
- // staff.yandex-team.ru login - only for passport users
- string staff_login = 1;
- // passport uid (external). default_uid for Yandex.Passport active multi-session.
- int64 passport_uid = 2;
- // passport login
- string login = 3;
- // For size-specific avatar rendering purposes https://avatars.mds.yandex.net/get-yapic/<avatarId>/<avatarSize>
- // see https://doc.yandex-team.ru/blackbox/reference/method-sessionid-response-json.html#method-sessionid-response-json__display-name
- string avatar_id = 4;
- // passport default email
- string email = 5;
- bool two_factor_authentication_enabled = 6;
- int64 karma = 7;
- repeated string phones = 8;
- Domain domain = 9;
- string login_id = 10;
-
- message Domain {
- string domain = 1;
- string domid = 2;
- bool hosted = 3;
- }
-}
-
-enum SubjectType {
- SUBJECT_TYPE_UNSPECIFIED = 0;
- USER_ACCOUNT = 1;
- SERVICE_ACCOUNT = 2;
- GROUP = 3;
- INVITEE = 4;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/CMakeLists.txt
deleted file mode 100644
index 3d82ddefb0..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/CMakeLists.txt
+++ /dev/null
@@ -1,59 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(priv-oauth-v1)
-set_property(TARGET priv-oauth-v1 PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET priv-oauth-v1 PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(priv-oauth-v1 PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(priv-oauth-v1 PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(priv-oauth-v1 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- iam-v1-token
- iam-v1-ts
- cloud-priv-oauth
- contrib-libs-protobuf
-)
-target_proto_messages(priv-oauth-v1 PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/claim_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/cloud_user.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/oauth_request.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/session_service.proto
-)
-target_proto_addincls(priv-oauth-v1
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(priv-oauth-v1
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(priv-oauth-v1
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/claim_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/claim_service.proto
deleted file mode 100644
index 938a2fdf07..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/claim_service.proto
+++ /dev/null
@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.oauth.v1;
-
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/oauth/claims.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1;oauth";
-option java_outer_classname = "PACS";
-
-service ClaimService {
- rpc Get (GetClaimsRequest) returns (GetClaimsResponse);
-}
-
-message GetClaimsRequest {
- repeated string subject_ids = 1 [(size) = "1-1000", (length) = "<=50"];
-}
-
-message GetClaimsResponse {
- repeated SubjectDetails subject_details = 2;
-}
-
-message SubjectDetails {
- SubjectClaims subject_claims = 1 [(required) = true];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/clients.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/clients.yaml
deleted file mode 100644
index 5ca0c87b57..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/clients.yaml
+++ /dev/null
@@ -1,72 +0,0 @@
-oauth_clients:
- yc.oauth.oauth_server:
- name: 'Yandex.Cloud OAuth Server'
- scopes: []
- yc.oauth.assembly_workshop:
- name: 'Yandex.Cloud OAuth Server'
- scopes: []
-
- yc.oauth.console:
- name: 'Yandex.Cloud Management Console'
- includedRoles:
- - resource-manager.clouds.owner
- scopes:
- - yc.organization-manager.federations.manage
- - yc.organization-manager.users.manage # invite users to ogranization from cloud ACL page
- - yc.organization-manager.users.get # suggest users
- permissions:
- - organization-manager.organizations.listAccessBindings # show inherited bindings from organization
- - certificate-manager.providers.use # permission to request/view internal certificates for YC developers
-
- yc.oauth.doubleconsole:
- name: 'DoubleCloud Management Console'
- includedRoles:
- - organization-manager.organizations.owner
- - resource-manager.clouds.owner
- scopes: []
-
- yc.oauth.mobile-app:
- name: 'Yandex.Cloud Mobile Application'
- scopes: []
- includedRoles:
- - resource-manager.clouds.owner
-
- yc.oauth.partners:
- name: 'Yandex.Cloud Partners Portal'
- includedRoles:
- - viewer
- - billing.clouds.owner
- - billing.accounts.admin
- - marketplace.editor
- scopes:
- - yc.support.tickets.manage
- - yc.resource-manager.clouds.accessBindings.manage # XXX temporary, see CLOUD-70244 and CLOUDBIZ-5425 for details
-
- yc.oauth.iap-ydb-viewer:
- name: 'Yandex.Cloud Identity-Aware Proxy for YDB Viewer'
- scopes: []
- includedRoles:
- - iap.web.user
- - ydb.viewer
-
- yc.oauth.query:
- name: 'Yandex.Cloud Query'
- includedRoles:
- - ydb.viewer
- - mdb.viewer
- - storage.viewer
- - yds.viewer
- - yq.admin
- - yq.internal.admin # to view additional internal data for developers
- scopes:
- - yc.iam.serviceAccounts.get
- - yc.iam.serviceAccounts.use
- - yc.resource-manager.folders.get
- - yc.resource-manager.clouds.get
- - yc.organization-manager.users.get
-
- yc.oauth.chats-api:
- name: 'Yandex.Cloud Support Chats'
- scopes:
- - yc.support.chats.manage
- - yc.support.tickets.manage
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/cloud_user.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/cloud_user.proto
deleted file mode 100644
index c16c47a25d..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/cloud_user.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.oauth.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1;oauth";
-option java_outer_classname = "Login";
-
-message CloudUserInfo {
- // corresponding to eulaAccepted field
- bool is_eula_accepted = 1 [deprecated = true];
- // corresponding to hasClouds field
- bool is_member_of_cloud = 2 [deprecated = true];
- // corresponding to canCreateClouds field
- bool can_create_cloud = 3;
- // corresponding to checks field
- CreateCloudRestrictions create_cloud_restrictions = 4;
-
- message CreateCloudRestrictions {
- bool has_phone = 1;
- bool has_email = 2;
- bool has_login = 3;
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/oauth_request.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/oauth_request.proto
deleted file mode 100644
index 693acbcad4..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/oauth_request.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.oauth.v1;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1;oauth";
-option java_outer_classname = "OAuthModel";
-
-// OAuth request
-message OAuthRequest {
- // client_id
- string client_id = 1 [(required) = true];
- repeated string scopes = 3;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/permissions.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/permissions.yaml
deleted file mode 100644
index 6d0cd6046b..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/permissions.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-permissions:
- internal.oauth.checkSession:
- description: 'Возможность проверять валидность per-service cookies'
- stage: GA
- visibility: internal
- resourceType: organization-manager.application
- internal.oauth.createSession:
- description: 'Возможность создавать per-service cookies'
- stage: GA
- visibility: internal
- resourceType: organization-manager.application
- internal.oauth.getUserAccountClaims:
- description: 'Get user accounts claims'
- stage: GA
- visibility: internal
- resourceType: iam.gizmo
- internal.oauth.checkPassportSession:
- description: 'Check passport cookies and issue IAM-token'
- stage: GA
- visibility: internal
- resourceType: organization-manager.application
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/roles.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/roles.yaml
deleted file mode 100644
index 17f8f44df2..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/roles.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-role_groups:
- oauth:
- name: OAuth roles
-
-roles:
- internal.sessionService:
- groups: [oauth]
- name: (Внутренняя) session service
- visibility: internal
- description: Роль для выписывания per-service cookie в session service (кросс-доменная аутентификация)
- permissions:
- - iam.userAccounts.issueCookie
- - iam.userAccounts.issueToken
- - iam.serviceAccounts.issueCookie
- - iam.serviceAccounts.issueToken
- - iam.subjects.issueCookie
- - iam.subjects.issueToken
-
- internal.sessionService.userAccountAgent:
- groups: [oauth]
- name: User Account Agent
- visibility: internal
- description: Внутренняя роль для просмотра пользовательских аккаунтов
- permissions:
- - internal.oauth.getUserAccountClaims
-
- internal.oauth.client:
- groups: [oauth]
- name: (Внутренняя) session service + oauth client
- visibility: internal
- description: Роль для выписывания и проверки per-service cookie конечными сервисами
- permissions:
- - internal.oauth.checkSession
- - internal.oauth.createSession
-
- internal.oauth.passportClient:
- groups: [oauth]
- name: (Внутренняя) session service
- visibility: internal
- description: Роль для выписывания и проверки per-service IAM-token конечными сервисами по Session_id куке Yandex.Passport
- permissions:
- - internal.oauth.checkPassportSession
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/scopes.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/scopes.yaml
deleted file mode 100644
index 4252f53200..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/scopes.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-scopes:
- openid:
- service: oauth
- name: ''
- # description полжен завершать предложение "приложение сможет" и отвечать на вопрос "сможет что делать?"
- description: 'аутентифицировать пользователя и получить его sub (subjectId)'
- visibility: public
- # сейчас нет возможности выдать пользователю права на чтение только своего профиля, поэтому список пермишенов пустой
- permissions: []
- profile:
- service: oauth
- name: 'получить доступ к основным данным профиля аутентифицированного пользователя (name, family_name, given_name, middle_name, nickname, preferred_username, profile, picture, website, gender, birthdate, zoneinfo, locale, updated_at)'
- description: ''
- visibility: public
- permissions: []
- email:
- service: oauth
- name: 'получить доступ к атрибутам email, email_verified аутентифицированного пользователя'
- description: ''
- visibility: public
- permissions: []
- address:
- service: oauth
- name: 'получить доступ к атрибуту address аутентифицированного пользователя'
- description: ''
- visibility: public
- permissions: []
- phone:
- service: oauth
- name: 'получить доступ к атрибутам phone_number, phone_number_verified аутентифицированного пользователя'
- description: ''
- visibility: public
- permissions: []
- offline_access:
- service: oauth
- name: 'получить refresh_token для аутентифицированного пользователя'
- description: ''
- visibility: public
- permissions: []
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/session_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/session_service.proto
deleted file mode 100644
index 97ed98e8a6..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1/session_service.proto
+++ /dev/null
@@ -1,153 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.oauth.v1;
-
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/iam/v1/token/iam_token.proto";
-import "yandex/cloud/priv/iam/v1/ts/iam_token_service_subject.proto";
-import "yandex/cloud/priv/oauth/claims.proto";
-import "yandex/cloud/priv/oauth/v1/cloud_user.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/oauth/v1;oauth";
-option java_outer_classname = "OAuth";
-
-service SessionService {
- // Verify the identity of a subject for services, authenticated via Yandex.Cloud IdP.
- // IAM-token authorization is required.
- //
- // gRPC error codes
- //
- // Unauthenticated: authorization iam_token are invalid or may have expired.
- // InvalidArgument: the provided cookies are invalid or may have expired.
- // Additional information can be found in details at AuthorizationRequired message - in this case user should be redirected to specified URL
- rpc Check (CheckSessionRequest) returns (CheckSessionResponse);
-
- // Verify the identity of a subject for services, authenticated via Yandex.ID (Yandex.Passport).
- // IAM-token authorization is required.
- // Usage of this API is limited and will be deprecated.
- //
- // gRPC error codes
- //
- // Unauthenticated: authorization iam_token are invalid or may have expired.
- // InvalidArgument: the provided cookies are invalid or may have expired.
- rpc CheckPassport (CheckPassportSessionRequest) returns (CheckPassportSessionResponse);
-
- // Create per-service session
- //
- // gRPC error codes
- // Unauthenticated: authorization iam_token are invalid or may have expired.
- // InvalidArgument: the provided access_token is invalid or may have expired.
- // Additional information can be found in details at AuthorizationRequired message - in this case user should be redirected to specified URL
- // FailedPrecondition: openid scope is missed for specified access_token
- rpc Create (CreateSessionRequest) returns (CreateSessionResponse);
-
- // Logout from parent session
- rpc Logout (LogoutRequest) returns (LogoutResponse);
-
- // Accept EULA
- rpc AcceptEula (AcceptEulaRequest) returns (AcceptEulaResponse);
-}
-
-message AcceptEulaRequest {
- // HTTP-header Cookie with required authentication cookie values (e.g. Session_id)
- string cookie_header = 1 [(length) = "<=32768", (sensitive) = true];
- // Service host address, for example "datalens.yandex.ru" or "tracker.yandex.com".
- // Used for Yandex.Passport cookie validation (Yandex.Passport cookie is TLD-specific)
- string host = 2 [(required) = true, (length) = "<=253"];
- YandexCloudAgreements cloud_agreements = 3;
-}
-
-message AcceptEulaResponse {
- YandexCloudAgreements cloud_agreements = 1;
-}
-
-// Yandex.Cloud agreements
-message YandexCloudAgreements {
- // current Yandex.Cloud EULA text is here https://yandex.ru/legal/cloud_termsofuse/
- bool eula = 1;
- bool privacy_policy = 2;
- // Deny receiving advertising and other informational messages from the company Yandex.Cloud LLC (OGRN 1187746678580).
- bool deny_notifications = 3;
-}
-
-message CheckSessionRequest {
- // HTTP-header Cookie with required per-service cookie values (e.g. yc_session)
- string cookie_header = 1 [(length) = "<=32768", (sensitive) = true];
- // Service host address, for example "datalens.yandex.ru" or "tracker.yandex.com".
- // Used for authorize_url TLD calculation, Yandex.Passport cookie revalidation (Yandex.Passport cookie is TLD-specific)
- string host = 2 [(length) = "<=253"];
- // If present - specified federation id should be used for authorization
- // otherwise authorization IdP calculated from cookies.
- string federation_id = 3 [(length) = "<=50"];
-}
-
-message CheckSessionResponse {
- // Authenticated subject claims.
- SubjectClaims subject_claims = 1 [(required) = true];
- // per-service cookie expiration time.
- google.protobuf.Timestamp expires_at = 2;
- CloudUserInfo cloud_user_info = 3;
- yandex.cloud.priv.iam.v1.IamToken iam_token = 4;
- // Yandex.Passport active multisession.
- PassportSession passport_session = 5;
-}
-
-message CheckPassportSessionRequest {
- // HTTP-header Cookie with required per-service cookie values (e.g. yc_session)
- string cookie_header = 1 [(length) = "<=32768", (sensitive) = true];
- // Service host address, for example "datalens.yandex.ru" or "tracker.yandex.com".
- // Used for authorize_url TLD calculation, Yandex.Passport cookie revalidation (Yandex.Passport cookie is TLD-specific)
- string host = 2 [(required) = true, (length) = "<=253"];
- // organization-manager.application ID that is used to authorize and issuer IAM-token
- string client_id = 3 [(required) = true, (length) = "<=50"];
-}
-
-message CheckPassportSessionResponse {
- // Authenticated subject claims.
- SubjectClaims subject_claims = 1 [(required) = true];
- yandex.cloud.priv.iam.v1.IamToken iam_token = 4;
-}
-
-message PassportSession {
- // Yandex.Passport active multisession user info (including default user)
- repeated YandexClaims users = 1;
-}
-
-message CreateSessionRequest {
- // access_token from successful token response, see https://openid.net/specs/openid-connect-core-1_0.html#TokenResponse for details.
- string access_token = 1 [(length) = "<=4096", (sensitive) = true];
- // Which hosts are allowed to receive the cookie. In general - application should not send this parameter.
- // Domain should match one of the client_id redirect_uri. Unmatched domain parameter is ignored.
- // see http://www.rfcreader.com/#rfc6265_line474 for details.
- string domain = 2 [(length) = "<=253"];
-}
-
-message CreateSessionResponse {
- // HTTP-header Set-Cookie for End-User with required per-service cookies, e.g. yc_session
- repeated string set_cookie_header = 1 [(sensitive) = true];
- // per-service cookie expiration time.
- google.protobuf.Timestamp expires_at = 2;
-}
-
-message LogoutRequest {
- // HTTP-header Cookie with required per-service cookie values (e.g. yc_session)
- string cookie_header = 1 [(length) = "<=32768", (sensitive) = true];
- // Which hosts are allowed to receive the cookie. In general - application should not send this parameter.
- // Domain should match one of the client_id redirect_uri. Unmatched domain parameter is ignored.
- // see http://www.rfcreader.com/#rfc6265_line474 for details.
- string domain = 2 [(length) = "<=253"];
-}
-
-message LogoutResponse {
- yandex.cloud.priv.iam.v1.ts.Subject subject = 1;
- // HTTP-header Set-Cookie for End-User with required per-service cookies, e.g. yc_session
- // Cloud-specific user authentication cookies should be removed by Set-Cookie header.
- repeated string set_cookie_header = 2 [(sensitive) = true];
-}
-
-message AuthorizationRequired {
- // authorize URL, e.g. URL for /authorize OpenID Connect endpoint.
- string authorize_url = 1 [(required) = true, (length) = "<=2048"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/operation/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/operation/CMakeLists.txt
deleted file mode 100644
index 2c46cc6a81..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/operation/CMakeLists.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-priv-operation)
-set_property(TARGET cloud-priv-operation PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-priv-operation PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-operation PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-operation PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(cloud-priv-operation PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-priv-operation PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/operation/operation.proto
-)
-target_proto_addincls(cloud-priv-operation
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-priv-operation
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(cloud-priv-operation
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/operation/operation.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/operation/operation.proto
deleted file mode 100644
index 0867840b50..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/operation/operation.proto
+++ /dev/null
@@ -1,28 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.operation;
-
-import "google/protobuf/any.proto";
-import "google/rpc/status.proto";
-import "google/protobuf/timestamp.proto";
-
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/operation;operation";
-option java_outer_classname = "PO";
-
-message Operation {
- string id = 1;
- string description = 2; // ex: Create VM, Stop VM, Delete Disk, Snapshot Disk, etc
- google.protobuf.Timestamp created_at = 3;
- string created_by = 4;
- google.protobuf.Timestamp modified_at = 5;
-
- bool done = 6;
-
- google.protobuf.Any metadata = 7;
-
- oneof result {
- google.rpc.Status error = 8; // for error and cancellation
- google.protobuf.Any response = 9; // for 'Create/Update' should contain entity
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/quota/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/quota/CMakeLists.txt
deleted file mode 100644
index 4e9abb7d04..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/quota/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-priv-quota)
-set_property(TARGET cloud-priv-quota PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-priv-quota PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-quota PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(cloud-priv-quota PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(cloud-priv-quota PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-priv-quota PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/quota/quota.proto
-)
-target_proto_addincls(cloud-priv-quota
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-priv-quota
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(cloud-priv-quota
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/quota/quota.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/quota/quota.proto
deleted file mode 100644
index 8e3200a024..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/quota/quota.proto
+++ /dev/null
@@ -1,63 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.quota;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/quota;quota";
-option java_outer_classname = "PQ";
-
-// Cloud service should implement grpc service with signature
-//
-//service QuotaService {
-// rpc Get (quota.GetQuotaRequest) returns (quota.Quota);
-//
-// rpc BatchUpdateMetric (quota.BatchUpdateQuotaMetricsRequest) returns (google.protobuf.Empty);
-//}
-
-message Quota {
- string cloud_id = 1;
- repeated QuotaMetric metrics = 2;
-}
-
-message QuotaMetric {
- string name = 1; // formatted as <domain>.<metric>.<unit>, e.g. mdb.hdd.size
- int64 value = 2 [deprecated=true]; // use 'usage' field instead
- int64 limit = 3;
- double usage = 4;
-}
-
-message MetricLimit {
- string name = 1;
- int64 limit = 2;
-}
-
-message GetQuotaRequest {
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message UpdateQuotaMetricRequest {
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- MetricLimit metric = 2;
-}
-
-message BatchUpdateQuotaMetricsRequest {
- string cloud_id = 1 [(required) = true, (length) = "<=50"];
- repeated MetricLimit metrics = 2;
-}
-
-message GetQuotaDefaultRequest {
-}
-
-message GetQuotaDefaultResponse {
- repeated MetricLimit metrics = 1;
-}
-
-message QuotaFailure {
- message Violation {
- QuotaMetric metric = 1;
- int64 required = 2; // new value for the MetricLimit.limit, so it is: old limit + delta
- }
- string cloud_id = 1;
- repeated Violation violations = 2;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/resources.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/resources.yaml
deleted file mode 100644
index 9a051f003a..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/resources.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-resources:
- root:
- parents: []
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/restrictions.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/restrictions.yaml
deleted file mode 100644
index fe8e05053b..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/restrictions.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-restrictions:
- blockPermissions:
- freeTier:
- denyAllPermissionsByDefault: true
-
- deletingContainer:
- denyAllPermissionsByDefault: true
-
- billSuspend:
- servicesToStop: ["*"]
- resourcesToStop: []
- stopDelay: PT0S
- deletionInitiationInterval: P57D
- deletionDelay: P3D
- denyAllPermissionsByDefault: true
-
- clientBlocking:
- servicesToStop: ["*"]
- stopDelay: PT0S
- denyAllPermissionsByDefault: true
-
- fraud:
- servicesToStop: ["*"]
- stopDelay: PT0S
- deletionInitiationInterval: P4D
- deletionDelay: P3D
- denyAllPermissionsByDefault: true
-
- elasticsearchSanctions:
- servicesToStop: ["managed-elasticsearch"]
- stopDelay: P7D
- denyAllPermissionsByDefault: false
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/roles.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/roles.yaml
deleted file mode 100644
index 091520149e..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/roles.yaml
+++ /dev/null
@@ -1,447 +0,0 @@
-role_groups:
- primitive:
- name: Primitive roles
- staff:
- name: Cloud Staff Roles
- infra:
- name: YC Infra roles
-
-roles:
-
- ##############################################################################
- # Primitive Roles
- ##############################################################################
-
- viewer:
- name: viewer
- groups: [primitive]
- description: доступ ко всем функциям в режиме чтения
- visibility: public
- includedRoles:
- - alb.viewer.pseudo
- - api-gateway.viewer
- - audit-trails.configViewer
- - audit-trails.viewer
- - billing.accounts.viewer
- - cdn.viewer
- - certificate-manager.viewer
- - cic.partnerViewer
- - cic.secretViewer
- - cic.viewer
- - cloudbeaver.viewer
- - compute.instanceGroups.viewer
- - compute.viewer
- - container-registry.viewer
- - data-transfer.viewer
- - dataproc.viewer
- - datasphere.user
- - dcnetwork.viewer # DoubleCloud Network
- - dns.viewer
- - gitlab.viewer
- - iam.viewer
- - iot.viewer
- - k8s.cluster-api.viewer
- - k8s.viewer
- - kms.viewer
- - load-balancer.viewer
- - load-balancer.viewer
- - loadtesting.viewer
- - lockbox.viewer
- - logging.reader
- - logging.viewer
- - marketplace.viewer
- - mdb.viewer
- - monitoring.viewer
- - organization-manager.viewer
- - quota-manager.viewer
- - resource-manager.viewer
- - serverless.containers.viewer
- - serverless.functions.viewer.pseudo
- - smart-captcha.viewer
- - storage.viewer
- - vdi.viewer
- - vpc.viewer
- - ydb.viewer
- - ydb.viewer.pseudo
- - yds.viewer
- - ymq.viewer
- - ymq.viewer.pseudo
- - yq.viewer
- scopes:
- - yc.datalens.view
- - yc.support.tickets.manage
- permissions:
- - cloudai.resources.{get,list}
- - logs.logEvents.read
-
- editor:
- name: 'editor'
- groups: [primitive]
- description: доступ ко всем функциям в режиме чтения и записи
- visibility: public
- includedRoles:
- - ai.speechkit-stt.user
- - ai.speechkit-tts.user
- - ai.translate.user
- - ai.vision.user
- - alb.editor.pseudo
- - api-gateway.editor
- - audit-trails.editor
- - billing.accounts.editor
- - cdn.editor
- - certificate-manager.editor
- - cic.editor
- - cic.partnerEditor
- - cic.secretEditor
- - cloudbeaver.editor
- - compute.editor
- - compute.instanceGroups.editor
- - compute.osLogin
- - container-registry.editor
- - data-transfer.editor
- - datalens.instances.user
- - dataproc.editor
- - dcnetwork.editor # DoubleCloud Network
- - dns.editor
- - gitlab.editor
- - iam.editor
- - iot.editor
- - k8s.cluster-api.cluster-admin
- - k8s.editor
- - kms.editor
- - loadtesting.editor
- - lockbox.editor
- - logging.editor
- - logging.writer
- - marketplace.editor
- - mdb.editor
- - monitoring.editor
- - organization-manager.editor
- - quota-manager.editor
- - resource-manager.editor
- - serverless.containers.editor
- - serverless.functions.editor.pseudo
- - smart-captcha.editor
- - storage.editor
- - vdi.editor
- - viewer
- - vpc.privateEditor.pseudo
- - vpc.publicEditor.pseudo
- - vpc.securityGroups.editor.pseudo
- - vpc.user
- - ydb.editor
- - yds.editor
- - ylb.networkLoadBalancers.publicEditor.pseudo
- - ylb.targetGroups.editor.pseudo
- - ymq.editor
- - yq.editor
- permissions:
- - cloudai.biometry.exec
- - cloudai.locator.exec
- - cloudai.predictor.exec
- - cloudai.resources.{create,delete,update}
- - cloudai.speller.exec
- - cloudai.vocabulary.exec
- - logs.logEvents.write
-
- admin:
- name: 'admin'
- groups: [primitive]
- description: доступ ко всем функциям в режиме чтения и записи, управление пользователями
- и политиками
- visibility: public
- includedRoles:
- - alb.admin.pseudo
- - api-gateway.admin
- - audit-trails.admin
- - billing.accounts.admin
- - cdn.admin
- - certificate-manager.admin
- - cloudbeaver.admin
- - compute.admin
- - compute.osAdminLogin
- - container-registry.admin
- - data-transfer.admin
- - datalens.instances.admin
- - dataproc.admin
- - datasphere.admin
- - dcnetwork.admin # DoubleCloud Network
- - dns.admin
- - editor
- - gitlab.admin
- - iam.admin
- - iot.admin
- - k8s.admin
- - k8s.cluster-api.cluster-admin
- - kms.admin
- - loadtesting.admin
- - lockbox.admin
- - logging.admin
- - mdb.admin
- - monitoring.admin
- - organization-manager.admin
- - quota-manager.admin
- - resource-manager.admin
- - serverless.containers.admin
- - serverless.functions.admin
- - smart-captcha.admin
- - storage.admin
- - vdi.admin
- - vpc.admin
- - ydb.admin
- - yds.admin
- - ymq.admin
- - yq.admin
-
- auditor:
- #это временная роль, которая сейчас по составу неправильная и в таком виде ее нельзя пускать дальше в прод, пока сервисы не готовы поддерживать новую сервисную роль самостоятельно https://st.yandex-team.ru/CLOUDBIZ-6178
- name: auditor
- groups: [primitive]
- description: доступ только к конфигурации и метадате
- visibility: internal
- includedRoles:
- - storage.configViewer
- - vpc.viewer
- - k8s.viewer
- - k8s.cluster-api.viewer
- - lockbox.viewer
- - kms.viewer
- - audit-trails.configViewer
- - load-balancer.viewer
- - certificate-manager.viewer
- - container-registry.viewer
- - iam.viewer
- - dns.viewer
- - monitoring.viewer
- permissions:
- - compute.console.getFolderStats
- - compute.console.getDiskLimits
- - compute.diskOperations.list
- - compute.disks.{get,list}
- - compute.filesystemOperations.list
- - compute.filesystems.{get,list}
- - compute.imageOperations.list
- - compute.images.getLatestByFamily
- - compute.images.{get,list}
- - compute.instanceOperations.list
- - compute.instances.{get,list}
- - compute.instanceGroups.{get,list}
- - compute.networkInterfaces.{get,list}
- - compute.operations.{get,list}
- - compute.placementGroups.{get,list,listInstances}
- - compute.placementGroupOperations.list
- - compute.platforms.get
- - compute.diskPlacementGroups.{get,list,listDisks}
- - compute.diskPlacementGroupOperations.list
- - compute.hostGroups.{get,list,listHosts,listInstances}
- - compute.hostGroupOperations.list
- - compute.quotas.getCloudLimit
- - compute.snapshotOperations.list
- - compute.snapshots.{get,list}
- - compute.zones.{get,list}
-
- ##############################################################################
- # Support Roles
- ##############################################################################
-
- support:
- groups: [staff]
- name: Техподдержка
- visibility: internal
- description: возможность проведения специфичных для support операций
- includedRoles:
- - alb.support
- - audit-trails.support
- - billing.support
- - cdn.support
- - certificate-manager.support
- - cic.support
- - cloudai.support
- - cloudbeaver.support
- - cloudsearch.cloudsIndex.support
- - compute.instanceGroups.support
- - compute.support
- - container-registry.support
- - data-transfer.support
- - datalens.support
- - disk-manager.support
- - dns.support
- - gitlab.support
- - iam.support
- - iot.support
- - k8s.support
- - kms.support
- - load-balancer.support
- - loadtesting.support
- - lockbox.support
- - logging.support
- - logs.support
- - marketplace.support
- - mdb.supportMdb
- - monitoring.support
- - nbs.support
- - quota-manager.support
- - serverless.support
- - storage.support
- - vdi.support
- - vpc.support
- - ydb.support
- - yds.support
- - ymq.support
- permissions: []
-
- supportAdmin:
- groups: [staff]
- name: Техподдержка
- visibility: internal
- description: доступ к методам удаления объектов и квотам compute
- includedRoles:
- - alb.supportAdmin
- - audit-trails.supportAdmin
- - billing.supportAdmin
- - cdn.supportAdmin
- - certificate-manager.supportAdmin
- - cic.supportAdmin
- - cloudai.supportAdmin
- - cloudbeaver.supportAdmin
- - compute.instanceGroups.supportAdmin
- - compute.supportAdmin
- - container-registry.supportAdmin
- - data-transfer.supportAdmin
- - datalens.supportAdmin
- - disk-manager.supportAdmin
- - dns.supportAdmin
- - gitlab.supportAdmin
- - iam.supportAdmin
- - iot.supportAdmin
- - k8s.supportAdmin
- - kms.supportAdmin
- - load-balancer.supportAdmin
- - loadtesting.supportAdmin
- - lockbox.supportAdmin
- - logging.supportAdmin
- - logs.supportAdmin
- - marketplace.supportAdmin
- - mdb.supportAdmin
- - monitoring.supportAdmin
- - nbs.supportAdmin
- - quota-manager.supportAdmin
- - serverless.supportAdmin
- - storage.supportAdmin
- - vdi.supportAdmin
- - vpc.supportAdmin
- - ydb.supportAdmin
- - yds.supportAdmin
- - ymq.supportAdmin
- permissions: []
-
- supportadmin:
- groups: [staff]
- name: Техподдержка
- visibility: internal
- description: DEPRECATED
- includedRoles:
- - alb.supportAdmin
- - audit-trails.supportAdmin
- - billing.supportAdmin
- - cdn.supportAdmin
- - certificate-manager.supportAdmin
- - cloudai.supportAdmin
- - compute.supportAdmin
- - container-registry.supportAdmin
- - data-transfer.supportAdmin
- - datalens.supportAdmin
- - disk-manager.supportAdmin
- - iam.supportAdmin
- - iot.supportAdmin
- - k8s.supportAdmin
- - kms.supportAdmin
- - load-balancer.supportAdmin
- - lockbox.supportAdmin
- - logs.supportAdmin
- - marketplace.supportAdmin
- - mdb.supportAdmin
- - monitoring.supportAdmin
- - nbs.supportAdmin
- - serverless.supportAdmin
- - storage.supportAdmin
- - vdi.supportAdmin
- - vpc.supportAdmin
- - ydb.supportAdmin
- - yds.supportAdmin
- - ymq.supportAdmin
- permissions: []
-
-
- ##############################################################################
- # On-call Roles
- ##############################################################################
-
- onCall:
- groups: [staff]
- name: Инженер on-call
- visibility: internal
- description: доступы для всех инженеров on-call
- includedRoles:
- - monitoring.viewer # все дежурные могут смотреть в мониторинге метрики любых облаков и ресурсов CLOUD-67217
- - cloudsearch.cloudsIndex.onCall # все дежурные могут смотреть в индекс по всем ресурсам
- permissions:
- - organization-manager.organizations.get
- - resource-manager.clouds.get
- - resource-manager.folders.get
-
-
- ##############################################################################
- # Internal Roles
- ##############################################################################
-
- compute.admin:
- groups: [compute, instance-group]
- name: Администратор Compute
- visibility: public
- description: доступ ко всем функциям compute, включая instance-group
- includedRoles:
- - compute.admin.withoutVpc.pseudo
- - compute.instanceGroups.admin
- - vpc.user
-
- internal.empty:
- groups: [infra]
- name: (Внутренняя) Пустая роль
- visibility: internal
- description: роль без полномочий для тестовых целей
- permissions: []
-
- internal.infra:
- groups: [infra]
- name: Роль для SA селфхоста
- visibility: internal
- description: ''
- permissions: []
-
- internal.teamcityagent:
- groups: [infra]
- name: (Внутренняя) Агент Teamcity
- visibility: internal
- description: доступ к Container Registry
- permissions:
- - container-registry.images.{create,update}
- - container-registry.images.{get,list}
- - container-registry.quotas.get
- - container-registry.registries.{get,list}
-
- internal.computehead:
- groups: [infra]
- name: "(Внутренняя) Роль для SA head'ов compute"
- visibility: internal
- description: ''
- includedRoles:
- - internal.compute.snapshotService.admin
- - internal.marketplacelicense
- - internal.disk-manager.user
- - internal.nbs.admin
- permissions:
- - vpcInternal.computeInternal.use
- - compute.worker.catchTasks
- - resource-manager.folders.get
- - resource-manager.clouds.get
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/sensitive.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/sensitive.proto
deleted file mode 100644
index be953a9dcb..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/sensitive.proto
+++ /dev/null
@@ -1,24 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv;cloud";
-
-enum SensitiveType {
- SENSITIVE_TYPE_UNSPECIFIED = 0;
- SENSITIVE_CRC = 1;
- SENSITIVE_IAM_TOKEN = 2;
- SENSITIVE_REMOVE = 3;
- SENSITIVE_YANDEX_PASSPORT_OAUTH_TOKEN = 4;
- SENSITIVE_IAM_COOKIE = 5;
-}
-
-extend google.protobuf.FieldOptions {
- // novikoff:
- // Sensitive fields are hidden in logs
- // For now could be applied only to string fields
- bool sensitive = 100601;
- SensitiveType sensitive_type = 100602;
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/CMakeLists.txt b/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/CMakeLists.txt
deleted file mode 100644
index 610138bc7c..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/CMakeLists.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(priv-servicecontrol-v1)
-set_property(TARGET priv-servicecontrol-v1 PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET priv-servicecontrol-v1 PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/private-api
-)
-target_include_directories(priv-servicecontrol-v1 PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_include_directories(priv-servicecontrol-v1 PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_link_libraries(priv-servicecontrol-v1 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-priv
- contrib-libs-protobuf
-)
-target_proto_messages(priv-servicecontrol-v1 PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/access_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/resource.proto
-)
-target_proto_addincls(priv-servicecontrol-v1
- ./cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/private-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(priv-servicecontrol-v1
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/private-api
-)
-target_proto_plugin(priv-servicecontrol-v1
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/access_service.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/access_service.proto
deleted file mode 100644
index 4f6e066134..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/access_service.proto
+++ /dev/null
@@ -1,255 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.servicecontrol.v1;
-
-import "google/protobuf/timestamp.proto";
-import "google/protobuf/any.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/priv/validation.proto";
-import "yandex/cloud/priv/sensitive.proto";
-import "yandex/cloud/priv/servicecontrol/v1/resource.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1;servicecontrol";
-
-service AccessService {
- // Verify the identity of a subject.
- //
- // gRPC error codes
- //
- // Ok: the provided credentials are valid
- // Unauthenticated: the provided credentials are invalid or may have expired
- // InvalidArgument: the client specified an invalid argument (please note that this applies to the request in itself,
- // not to the content of the request, i.e. you will get the InvalidArgument error if the message
- // size exceeds the server limit but Unauthenticated if the token format is not recognized)
- // Unavailable: the service is currently unavailable, the client should retry again
- // Internal: the service is broken
- //
- // Please note that these do not include client-side errors (e.g. Cancelled, DeadlineExceeded, etc.)
- rpc Authenticate (AuthenticateRequest) returns (AuthenticateResponse);
-
- // Check if a subject is allowed to perform an action. This also authenticates the subject if any credentials are
- // passed as an identity.
- //
- // gRPC error codes
- //
- // Ok: the provided credentials (if any) are valid and the subject has permissions to access the
- // specified resource
- // Unauthenticated: the provided credentials are invalid or may have expired
- // PermissionDenied: the subject does not have permissions to access the specified resource
- // InvalidArgument: the client specified an invalid argument (please note that this applies to the request in itself,
- // not to the content of the request, i.e. you will get the InvalidArgument error if the message
- // size exceeds the server limit or the specified permission does not exist but Unauthenticated if
- // the token format is not recognized)
- // Unavailable: the service is currently unavailable, the client should retry again
- // Internal: the service is broken
- //
- // Please note that these do not include client-side errors (e.g. Cancelled, DeadlineExceeded, etc.)
- rpc Authorize (AuthorizeRequest) returns (AuthorizeResponse);
-
- // Similar to Authorize, but requests multiple actions for one subject.
- //
- // gRPC error codes will be the same, except for these cases:
- // - An Unauthenticated error of BulkAuthorizeRequest.identity is returned in
- // BulkAuthorizeResponse.unauthenticated_error.
- // - All PermissionDenied of BulkAuthorizeRequest.authorizations are returned in
- // BulkAuthorizeResponse.results.
- //
- // You can control the information returned in BulkAuthorizeResponse.results with:
- // - result_filter : return all errors (ALL_FAILED) or only the first one (FIRST_FAILED), if any.
- // - result_mask : You can choose the fields returned (all by default),
- // from the fields in BulkAuthorizeResponse.Result.
- //
- rpc BulkAuthorize (BulkAuthorizeRequest) returns (BulkAuthorizeResponse);
-}
-
-message AuthenticateRequest {
- oneof credentials {
- option (exactly_one) = true;
-
- // IAM-token obtained from the IAM Token Service.
- // The server response for an empty IAM token is UNAUTHENTICATED
- string iam_token = 1 [(length) = "<=1024", (sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
-
- // AWS-compatible signature.
- AccessKeySignature signature = 2;
-
- // API key.
- // The server response for an empty API key is UNAUTHENTICATED
- string api_key = 3;
-
- // IAM-cookie.
- // The server response for an empty IAM cookie is UNAUTHENTICATED
- string iam_cookie = 4 [(sensitive) = true, (sensitive_type) = SENSITIVE_IAM_COOKIE];
- }
-}
-
-message AuthenticateResponse {
- Subject subject = 1;
-}
-
-message AuthorizeRequest {
- oneof identity {
- option (exactly_one) = true;
-
- Subject subject = 1;
-
- // IAM-token obtained from the IAM Token Service.
- // The server response for an empty IAM token is UNAUTHENTICATED
- string iam_token = 2 [(length) = "<=1024", (sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
-
- // AWS-compatible signature.
- AccessKeySignature signature = 3;
-
- // API key.
- // The server response for an empty API key is UNAUTHENTICATED
- string api_key = 6;
- }
-
- string permission = 4 [(required) = true, (length) = "<=128"];
-
- // A resource to authorize access to. This may also include a service-specific hierarchy of the resource, usually
- // ends with resource-manager.folder.
- //
- // Examples:
- // (resource-manager.folder, b1gn3enigctah04o0fkb)
- // (billing.account, b1gqql62454n46tboesn)
- // (compute.instance, b1gqqhvc4fg65mkrefs8), (resource-manager.folder, b1gn3enigctah04o0fkb)
- // (resource-manager.cloud, aje56o8prppkrpaiuoc6)
- // (my-service.instance, b1gqqepv0upu57issrog), (resource-manager.cloud, aje56o8prppkrpaiuoc6)
- repeated Resource resource_path = 5 [(size) = ">0"];
-}
-
-message AuthorizeResponse {
- Subject subject = 1;
-
- // Full path to the resource.
- repeated Resource resource_path = 2;
-}
-
-message BulkAuthorizeRequest {
- oneof identity {
- option (exactly_one) = true;
-
- Subject subject = 1;
-
- string iam_token = 2 [(length) = "<=1024", (sensitive) = true, (sensitive_type) = SENSITIVE_IAM_TOKEN];
-
- AccessKeySignature signature = 3;
-
- string api_key = 4;
- }
-
- oneof authorizations {
- option (exactly_one) = true;
-
- Actions actions = 5;
-
- ActionMatrix action_matrix = 6;
- }
-
- ResultFilter result_filter = 7;
-
- google.protobuf.FieldMask result_mask = 8;
-
- message Action {
- repeated Resource resource_path = 1 [(size) = "1-128"];
-
- string permission = 2 [(required) = true, (length) = "<=128"];
- }
-
- message Actions {
- repeated Action items = 1 [(size) = "1-1000"];
- }
-
- // Cross product of paths and permissions (represents N*M actions, N*M <= 1000).
- message ActionMatrix {
- repeated ResourcePath resource_paths = 2 [(size) = "1-1000"];
-
- repeated string permissions = 1 [(size) = "1-1000", (length) = "<=128"];
- }
-
- enum ResultFilter {
- RESULT_FILTER_UNSPECIFIED = 0;
- FIRST_FAILED = 1;
- ALL_FAILED = 2;
- }
-}
-
-message BulkAuthorizeResponse {
- Subject subject = 1;
-
- Error unauthenticated_error = 2;
-
- Results results = 3;
-
- message Results {
- repeated Result items = 1;
- }
-
- message Result {
- string permission = 1;
-
- repeated Resource resource_path = 2;
-
- Error permission_denied_error = 3;
- }
-
- message Error {
- string message = 1;
-
- repeated google.protobuf.Any details = 2;
- }
-}
-
-message AccessKeySignature {
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
- string string_to_sign = 2 [(required) = true, (length) = "<=8192"];
- string signature = 3 [(required) = true, (length) = "<=128", (sensitive) = true, (sensitive_type) = SENSITIVE_CRC];
-
- oneof parameters {
- option (exactly_one) = true;
-
- Version2Parameters v2_parameters = 4;
- Version4Parameters v4_parameters = 5;
- }
-
- message Version2Parameters {
- SignatureMethod signature_method = 1;
-
- enum SignatureMethod {
- SIGNATURE_METHOD_UNSPECIFIED = 0;
- HMAC_SHA1 = 1;
- HMAC_SHA256 = 2;
- }
- }
-
- message Version4Parameters {
- google.protobuf.Timestamp signed_at = 1 [(required) = true];
- string service = 2 [(required) = true, (length) = "<=64"];
- string region = 3 [(required) = true, (length) = "<=32"];
- }
-}
-
-message Subject {
- oneof type {
- option (exactly_one) = true;
-
- UserAccount user_account = 1;
- ServiceAccount service_account = 2;
- AnonymousAccount anonymous_account = 3;
- }
-
- message UserAccount {
- string id = 1 [(required) = true, (length) = "<=50"];
- string federation_id = 2 [(length) = "<=50"];
- }
-
- message ServiceAccount {
- string id = 1 [(required) = true, (length) = "<=50"];
- string folder_id = 2 [(length) = "<=50"];
- }
-
- // Use this if you want to check if an unauthenticated subject is allowed to access a resource.
- message AnonymousAccount {
- }
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/resource.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/resource.proto
deleted file mode 100644
index 4ad3293eb1..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1/resource.proto
+++ /dev/null
@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv.servicecontrol.v1;
-
-import "yandex/cloud/priv/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv/servicecontrol/v1;servicecontrol";
-option java_outer_classname = "PR";
-
-message Resource {
- string id = 1 [(required) = true, (length) = "<=50"];
-
- // The type of the resource, e.g. resource-manager.folder, billing.account, compute.snapshot, etc.
- string type = 2 [(required) = true, (length) = "<=64"];
-}
-
-message ResourcePath {
- repeated Resource path = 1 [(size) = "1-128"];
-}
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/services.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/services.yaml
deleted file mode 100644
index ef7524a0b4..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/services.yaml
+++ /dev/null
@@ -1,186 +0,0 @@
-# one of the following https://cloud.yandex.ru/services#_all
-services:
- dwh:
- name: Yandex Cloud DWH (Internal)
- description: Облачный DWH
- resource-manager:
- name: Yandex Resource Manager
- description: Управление ресурсами в каталогах и облаках
- storage:
- name: Yandex Object Storage
- description: Масштабируемое хранилище данных
- aliases: [s3]
- ai:
- name: Yandex SpeechKit, Yandex Translate, Yandex Vision
- description: Комплекс технологий распознавания и синтеза речи, машинный перевод, анализ изображений
- aliases: [cloudai]
- certificate-manager:
- name: Yandex Certificate Manager
- description: Управление TLS-сертификатами
- cdn:
- name: Yandex Cloud CDN
- description: Сервис CDN
- compute:
- name: Yandex Compute Cloud
- description: Виртуальные машины и диски
- container-registry:
- name: Yandex Container Registry
- description: Управление Docker-образами
- datalens:
- name: Yandex DataLens
- description: Визуализация и анализ данных
- datasphere:
- name: Yandex DataSphere
- description: Платформа для ML-разработчиков
- dns:
- name: Yandex Cloud DNS
- description: ДНС
- iam:
- name: Yandex Identity and Access Management
- description: Идентификация и контроль доступа к облачным ресурсам
- iot:
- name: Yandex IoT Core
- description: Решения для интернета вещей
- managed-kubernetes:
- name: Yandex Managed Service for Kubernetes
- description: Управление кластерами Kubernetes
- aliases: [k8s]
- kms:
- name: Yandex Key Management Service
- description: Управление ключами шифрования
- ylb:
- name: Yandex Load Balancer
- description: Сетевые балансировщики нагрузки
- aliases: [load-balancer]
- mdb:
- name: Yandex Managed Data Bases
- description: Управление базами данных
- instance-group:
- name: Yandex Instance Groups
- description: Группы виртуальных машин
- monitoring:
- name: Yandex Monitoring
- description: Сбор и визуализация метрик
- serverless:
- name: Yandex Cloud Functions
- description: Запуск вашего кода в виде функции
- api-gateway:
- name: Yandex API Gateway
- description: Cервис для управления API-шлюзами
- vpc:
- name: Yandex Virtual Private Cloud
- description: Управление сетями в облаке
- cic:
- name: Yandex Cloud Interconnect
- description: Связность между виртуальными сетями и on-premise сетями
- ydb:
- name: Yandex Database
- description: Распределённая отказоустойчивая NewSQL СУБД
- yds:
- name: Yandex Data Streams
- description: Отказоустойчивый сервис для передачи потоков данных
- ymq:
- name: Yandex Message Queue
- description: Очереди для обмена сообщениями между приложениями
- marketplace:
- name: Yandex Cloud Marketplace
- description: ''
- alb:
- name: Load Balancer API
- description: ''
- oauth:
- name: Yandex Cloud OAuth 2.0 & OpenID Connect server
- description: ''
- billing:
- name: Billing (internal)
- description: ''
- support:
- name: Support (internal)
- description: ''
- yq:
- name: Yandex Query
- description: Сервис безсерверной аналитики
- data-transfer:
- name: Data Transfer
- description: Сервис без серверного трансфера данных
- dataproc:
- name: Data Proc
- description: Управление кластерами Apache Hadoop и Apache Spark
- organization-manager:
- name: Organization Manager
- description: Управление Организациями
- vulnerability-scanner:
- name: Vulnerability Scanner (internal)
- description: Сервис сканирования образов на уязвимости
- lockbox:
- name: Yandex Lockbox
- description: Управление секретами
- managed-kafka:
- name: Yandex Managed Service for Apache Kafka
- description: Управление кластерами Apache Kafka
- managed-elasticsearch:
- name: Yandex Managed Service for Elasticsearch
- description: Управление кластерами Elasticsearch
- managed-sqlserver:
- name: Yandex Managed Service for Microsoft SQL Server
- description: Управление кластерами Microsoft SQL Server
- logs:
- name: Yandex Cloud Logs
- description: Запись, хранение, чтение и настройка логов
- logging:
- name: Yandex Cloud Logging
- description: Запись, хранение, чтение и настройка логов
- audit-trails:
- name: Yandex Audit Trails
- description: Сервис аудитных логов
- bastion:
- name: Bastion (internal)
- description: Сервис доступа по ssh
- loadtesting:
- name: Loadtesting
- description: Сервис нагрузочного тестирования
- vdi:
- name: Virtual Desktop Infrastructure
- description: Виртуальные рабочие места
- uk8s:
- name: Underlay K8S
- description: Kubernetes для железных хостов
- managed-gitlab:
- name: Yandex Managed Service for Gitlab
- description: Сервис управления Git-репозиториями
- aliases: [gitlab]
- quota-manager:
- name: Quota Manager
- description: Сервис просмотра квот и управление запросами на изменение квот
- cloudsearch:
- name: YC.CloudSearch
- description: Сервис поиска по облакам
- cloudbeaver:
- name: CloudBeaver
- description: Управление базами данных через web-интерфейс
- smart-captcha:
- name: Yandex SmartCaptcha
- description: Сервис для встраивания капчи
-
-#DoubleCloud services
- dcnetwork:
- name: DoubleCloud Network Service
- description: Управление сетями в DoubleCloud
-
-#Internal services
- backoffice:
- name: YC.Backoffice
- description: Внутренняя админка
-
- maintenance:
- name: Maintenance Service
- description: Обслуживание облачных сервисов
-
-#Collaboration services
- tracker:
- name: Yandex Tracker
- description: Яндекс Трекер, сервис для совместной работы и организации процессов в компании
-
- wiki:
- name: Yandex Wiki
- description: Яндекс Вики, сервис для ведения базы знаний
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/stages.yaml b/cloud/bitbucket/private-api/yandex/cloud/priv/stages.yaml
deleted file mode 100644
index 991fef1043..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/stages.yaml
+++ /dev/null
@@ -1,173 +0,0 @@
-stages:
- # Internal
- - GA
- - TEST_ALPHA # https://st.yandex-team.ru/CLOUD-36203
-
- # Resource reaper flags https://st.yandex-team.ru/CLOUD-42619
- - REAPER_NO_DELAY_FLAG
- - REAPER_WHITELIST_FLAG
-
- # MDB feature flags
- - MDB_ALLOW_DECOMMISSIONED_ZONE_USE
- - MDB_CLICKHOUSE_FAST_OPS
- - MDB_CLICKHOUSE_SHARDING
- - MDB_CLICKHOUSE_UNLIMITED_SHARD_COUNT
- - MDB_CLICKHOUSE_UPGRADE
- - MDB_CLICKHOUSE_CLOUD_STORAGE
- - MDB_CLICKHOUSE_CLOUD_STORAGE_HA
- - MDB_CLICKHOUSE_KEEPER
- - MDB_CLICKHOUSE_SQL_MANAGEMENT
- - MDB_CLICKHOUSE_TESTING_VERSIONS
- - MDB_DATAPROC_INSTANCE_GROUPS
- - MDB_DATAPROC_MANAGER
- - MDB_DATAPROC_AUTOSCALING
- - MDB_DATAPROC_ALLOW_DEPRECATED_VERSIONS
- - MDB_HADOOP_ALPHA
- - MDB_HADOOP_GPU
- - MDB_LOCAL_DISK_RESIZE
- - MDB_MONGODB_40
- - MDB_MONGODB_4_2
- - MDB_MONGODB_4_2_RS_UPGRADE
- - MDB_MONGODB_4_2_SHARDED_UPGRADE
- - MDB_MONGODB_EXTENDEDS
- - MDB_MONGODB_FAST_OPS
- - MDB_MONGODB_UNLIMITED_SHARD_COUNT
- - MDB_MYSQL_80
- - MDB_MYSQL_FAST_OPS
- - MDB_NETWORK_DISK_NO_STOP_RESIZE
- - MDB_NETWORK_DISK_TRUNCATE
- - MDB_POSTGRESQL_10_1C
- - MDB_POSTGRESQL_11
- - MDB_POSTGRESQL_11_1C
- - MDB_POSTGRESQL_12_1C
- - MDB_POSTGRESQL_13
- - MDB_POSTGRESQL_FAST_OPS
- - MDB_GREENPLUM_CLUSTER
- - MDB_REDIS
- - MDB_REDIS_FAST_OPS
- - MDB_REDIS_SHARDING
- - MDB_KAFKA_CLUSTER
- - MDB_KAFKA_CONNECT
- - MDB_SQLSERVER_CLUSTER
- - MDB_SQLSERVER_ALLOW_DEV
- - MDB_SQLSERVER_ALLOW_17_19
- - MDB_ELASTICSEARCH_CLUSTER
- - MDB_ELASTICSEARCH_ALLOW_UNLIMITED_HOSTS
- - MDB_DATAPROC_UI_PROXY
- - MDB_MONGODB_RS_PITR
- - MDB_ALLOW_NETWORK_SSD_NONREPLICATED
- - MDB_DATAPROC_IMAGE_1_3
- - MDB_MONGODB_INFRA_CFG
- - MDB_MONGODB_4_4
- - MDB_MONGODB_4_4_RS_UPGRADE
- - MDB_MONGODB_4_4_SHARDED_UPGRADE
- - MDB_MONGODB_SHARDED_PITR
- - MDB_MONGODB_BACKUP_SERVICE
- - MDB_FORCE_UNSAFE_RESIZE
- - MDB_MONGODB_RESTORE_WITHOUT_REPLAY
- - MDB_FLAVOR_80_512
- - MDB_DATAPROC_IMAGE_2_0 # https://st.yandex-team.ru/MDB-8214
- - MDB_MONGODB_ALLOW_DEPRECATED_VERSIONS
- - MDB_MONGODB_PERF_DIAG
- - MDB_V3GEN # https://st.yandex-team.ru/MDB-12883
- - MDB_SQLSERVER_TWO_NODE_CLUSTER # https://st.yandex-team.ru/MDB-13389
- - MDB_DEDICATED_HOSTS # https://st.yandex-team.ru/CLOUDFRONT-9054
- - MDB_MONGODB_ENTERPRISE # https://st.yandex-team.ru/MDB-16140
-
- # Other services feature flags
- - KMS_STRONG_ENCRYPTION # https://st.yandex-team.ru/CLOUD-34154
- - KMS_HSM_KEYS # https://st.yandex-team.ru/CLOUD-44371
- - SOFTWARE_ACCELERATED_NETWORK_ALPHA # https://st.yandex-team.ru/CLOUDFRONT-2752
- - VPC_DISABLE_ROUTE_TABLES_SUBNETS_CHECK # https://st.yandex-team.ru/CLOUD-30129
- - STORAGE_ENABLE_ENCRYPTION # https://st.yandex-team.ru/MDS-11418
- - STORAGE_ENABLE_SELECT # https://st.yandex-team.ru/MDS-13861
- - CDN_ENABLE_ACCESS # https://st.yandex-team.ru/CLOUD-63401
- - DATASPHERE_CUSTOM_LM_LEARNING # https://st.yandex-team.ru/CLOUD-52048
- - DATA_TRANSFER_LOGBROKER
- - DATA_TRANSFER_LOGFELLER
- - DATA_TRANSFER_MONGODB
- - DATA_TRANSFER_CLICKHOUSE
- - DATA_TRANSFER_YDB
- - DATA_TRANSFER_YDS # https://st.yandex-team.ru/TM-1460
- - DATA_TRANSFER_KAFKA # https://st.yandex-team.ru/TM-2041
- - DATA_TRANSFER_OBJECT_STORAGE # https://st.yandex-team.ru/TM-1747
- - DATA_TRANSFER_ORACLE # https://st.yandex-team.ru/TM-643
- - DATA_TRANSFER_POSTGRESQL_TO_YDB #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_MYSQL_TO_YDB #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_POSTGRESQL_TO_OBJECT_STORAGE #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_MYSQL_TO_OBJECT_STORAGE #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_MONGO_TO_OBJECT_STORAGE #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_ORACLE_TO_POSTGRESQL #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_ORACLE_TO_CLICKHOUSE #https://st.yandex-team.ru/TM-2477
- - DATA_TRANSFER_YC_FROM_LOGBROKER #https://st.yandex-team.ru/TM-2563
- - DATA_TRANSFER_YC_TO_LOGBROKER #https://st.yandex-team.ru/TM-2563
- - DATA_TRANSFER_AIRBYTE #https://st.yandex-team.ru/TM-2899
- - DATA_TRANSFER_GREENPLUM # https://st.yandex-team.ru/TM-3280
- - DATA_TRANSFER_DEBEZIUM # https://st.yandex-team.ru/TM-3218
- - DATASPHERE_GENERAL_NIRVANA_GRAPHS # https://st.yandex-team.ru/CLOUD-62288
- - DATASPHERE_CELL_DEPLOYMENT
- - IAM_FOLDER_SAML_FEDERATIONS # iam SAML-federations in folder https://st.yandex-team.ru/CLOUD-69157
- - IAM_ACCESS_KEY_PREFIX # https://st.yandex-team.ru/CLOUD-80368
- - SERVERLESS_FUNCTIONS_PROVISION_ALPHA # https://st.yandex-team.ru/CLOUD-71805
- - LOADTESTING_PREVIEW_ALPHA # https://st.yandex-team.ru/CLOUDLOAD-77
- - DATASPHERE_TAAS # https://st.yandex-team.ru/CLOUD-80447
- - DATASPHERE_IDE_ON_SERVANT # https://st.yandex-team.ru/CLOUD-85578
- - DATASPHERE_DISK_VERSIONING # https://st.yandex-team.ru/CLOUD-93160
- - TRIGGERS_CLOUD_LOGS_ALPHA # https://st.yandex-team.ru/CLOUD-80291
- - TRIGGERS_MAIL_ALPHA # https://st.yandex-team.ru/CLOUD-95233
- - SMART_CAPTCHA_ALPHA # https://st.yandex-team.ru/CAPTCHA-2599
- - SERVERLESS_FUNCTIONS_SECRETS_ALPHA # https://st.yandex-team.ru/CLOUD-52449
- - IOT_BROKER_ALPHA # https://st.yandex-team.ru/CLOUD-95928
-
- # Services in development
- - APIGW_ALPHA # https://st.yandex-team.ru/CLOUD-31754
- - SERVERLESS_VPC # https://st.yandex-team.ru/CLOUD-40827
- - CERTIFICATE_MANAGER_ALPHA # https://st.yandex-team.ru/CLOUD-35187
- - CUSTOM_FQDN_INSTANCES_ALPHA # https://st.yandex-team.ru/CLOUD-15718
- - EGRESS_NAT_ALPHA # https://st.yandex-team.ru/CLOUD-35281
- - CIC_ALPHA # https://st.yandex-team.ru/CLOUD-74655
- - KMS_ALPHA # https://st.yandex-team.ru/CLOUD-26559
- - ML_PLATFORM_ALPHA # https://st.yandex-team.ru/CLOUD-35731
- - MULTI_INTERFACE_INSTANCES_ALPHA # https://st.yandex-team.ru/CLOUD-15718
- - UDP_LISTENER_YLB_ALPHA # https://st.yandex-team.ru/CLOUD-34630
- - YDB_ALPHA # https://st.yandex-team.ru/CLOUD-24439
- - COMPUTE_OSLOGIN_ALPHA # https://st.yandex-team.ru/CLOUD-21127
- - DATA_TRANSFER_ALPHA #https://st.yandex-team.ru/CLOUD-40128
- - DNS_ALPHA #https://st.yandex-team.ru/CLOUD-42207
- - MDBPROXY_ALPHA #https://st.yandex-team.ru/CLOUD-38110
- - IMAGE_SCANNER_ALPHA #https://st.yandex-team.ru/CLOUD-34754
- - LOCKBOX_ALPHA #https://st.yandex-team.ru/CLOUD-46437
- - ALB_ALPHA #https://st.yandex-team.ru/CLOUD-58061
- - LOGGING_ALPHA # https://st.yandex-team.ru/CLOUD-62388
- - LOGS_ALPHA # deprecated, use LOGGING_ALPHA
- - AUDIT_TRAILS_ALPHA # https://st.yandex-team.ru/CLOUD-51927
- - RESOURCEMANAGER_DELETE_ALPHA # https://st.yandex-team.ru/CLOUD-63858
- - DATA_STREAMS_ALPHA # https://st.yandex-team.ru/CLOUD-68113
- - SERVERLESS_CONTAINERS_ALPHA # https://st.yandex-team.ru/CLOUD-72622
- - SERVERLESS_CONTAINERS_PROVISION_ALPHA # https://st.yandex-team.ru/CLOUD-85491
- - K8S_MARKETPLACE_ALPHA #https://st.yandex-team.ru/CLOUD-72635
- - K8S_MARKETPLACE_FRONT_ALPHA # https://st.yandex-team.ru/CLOUD-84678
- - VDI_ALPHA #https://st.yandex-team.ru/CLOUDBIZ-5195
- - MANAGED_GITLAB_ALPHA # https://st.yandex-team.ru/CLOUD-71805
- - CLOUD_BEAVER_ALPHA # https://st.yandex-team.ru/CLOUD-87921
-
- # VPC
- - INTERNAL_YLB_ALPHA # https://st.yandex-team.ru/CLOUD-18755
- - VPC_SG_ALPHA # https://st.yandex-team.ru/CLOUD-35341
- - VPC_IPV6_ALPHA # https://st.yandex-team.ru/CLOUD-40113
- - VPC_YANDEX_SERVICES # https://st.yandex-team.ru/CLOUD-62110
-
- # Marketplace
- - DISABLE_PLACEMENT_HINT_WINDOWS_DC # https://st.yandex-team.ru/CLOUD-82570
- - DISABLE_PLACEMENT_HINT_WINDOWS_DC_SQL_ENTERPRISE # https://st.yandex-team.ru/CLOUD-82570
- - YAGA_MARKETPLACE_ALPHA # https://st.yandex-team.ru/CLOUD-85253
- - MARKETPLACE_MONGODB_ENTERPRISE # https://st.yandex-team.ru/CLOUD-80323
-
- # Yandex Query
- - YANDEX_QUERY_ALPHA # https://st.yandex-team.ru/CLOUD-79231
- - YANDEX_QUERY_CONNECTION_CH
- - YANDEX_QUERY_CONNECTION_YDB
-
- # Outdated, don't use these (Stages CAN'T BE REMOVED)
- - VPC_DHCP_OPTIONS_ALPHA
-
diff --git a/cloud/bitbucket/private-api/yandex/cloud/priv/validation.proto b/cloud/bitbucket/private-api/yandex/cloud/priv/validation.proto
deleted file mode 100644
index daec144396..0000000000
--- a/cloud/bitbucket/private-api/yandex/cloud/priv/validation.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.priv;
-
-import "google/protobuf/descriptor.proto";
-
-// See yandex/cloud/priv/example/v1alpha/validation_example.proto for usage examples.
-option go_package = "a.yandex-team.ru/cloud/bitbucket/private-api/yandex/cloud/priv;cloud";
-
-extend google.protobuf.OneofOptions {
- bool exactly_one = 100400;
-}
-
-extend google.protobuf.FieldOptions {
- bool required = 100501;
- string pattern = 100502;
- string value = 100503;
- string size = 100504;
- string length = 100505;
- bool unique = 100506;
- MapKeySpec map_key = 100510;
- string bytes = 100511;
-}
-
-message MapKeySpec {
- string value = 1;
- string pattern = 2;
- string length = 3;
-}
diff --git a/cloud/bitbucket/public-api/AUTHORS b/cloud/bitbucket/public-api/AUTHORS
deleted file mode 100644
index 26d20b4d17..0000000000
--- a/cloud/bitbucket/public-api/AUTHORS
+++ /dev/null
@@ -1,32 +0,0 @@
-The following authors have created the source code of "Yandex.Cloud API" published and distributed by YANDEX LLC as the owner:
-
-Alexander Burmak <alex-burmak@yandex-team.ru>
-Alexander Kirakozov <akirakozov@yandex-team.ru>
-Alexander Klyuev <wizard@yandex-team.ru>
-Alexander Serkov <alxn1@yandex-team.ru>
-Alexey Baranov <baranovich@yandex-team.ru>
-Alexey Zamulla <zamulla@yandex-team.ru>
-Alexey Zasimov <zasimov-a@yandex-team.ru>
-Amy Krishnevsky <krishnevsky@yandex-team.ru>
-Anastasia Karavaeva <dottir@yandex-team.ru>
-Andrey Polyakov <koshachy@yandex-team.ru>
-Damir Makhmutov <yesworld@yandex-team.ru>
-Danila Diugurov <terry@yandex-team.ru>
-David Lanchava <landavid@yandex-team.ru>
-Elena Ilycheva <eilycheva@yandex-team.ru>
-Evgeny Arhipov <arhipov@yandex-team.ru>
-Luba Grinkevich <luba239@yandex-team.ru>
-Maxim Kolganov <manykey@yandex-team.ru>
-Mikhail Goncharov <migelle@yandex-team.ru>
-Nikolay Amelichev <entropia@yandex-team.ru>
-Pavel Fomin <vaccarium@yandex-team.ru>
-Rurik Krylov <rurikk@yandex-team.ru>
-Evgeny Dyukov <secwall@yandex-team.ru>
-Sergey Kanunnikov <skanunnikov@yandex-team.ru>
-Sergey Kiselev <intr13@yandex-team.ru>
-Sergey Sytnik <ssytnik@yandex-team.ru>
-Stanislav Ievlev <sievlev@yandex-team.ru>
-Vasilii Briginets <0x40@yandex-team.ru>
-Vlad Arkhipov <potamus@yandex-team.ru>
-Vladimir Borodin <d0uble@yandex-team.ru>
-Vladimir Skipor <skipor@yandex-team.ru> \ No newline at end of file
diff --git a/cloud/bitbucket/public-api/LICENSE b/cloud/bitbucket/public-api/LICENSE
deleted file mode 100644
index 0cd74fabf0..0000000000
--- a/cloud/bitbucket/public-api/LICENSE
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2018 YANDEX LLC
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff --git a/cloud/bitbucket/public-api/README.md b/cloud/bitbucket/public-api/README.md
deleted file mode 100644
index 43465310ef..0000000000
--- a/cloud/bitbucket/public-api/README.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# Yandex.Cloud API
-
-This repository contains the interface definitions of Yandex.Cloud API.
-
-For more details see [documentation](https://cloud.yandex.com/docs).
-
-## Overview
-
-You can access Yandex.Cloud by using REST or GRPC API.
-
-Yandex.Cloud API uses Protocol Buffers to define API interfaces and data structures. These definitions are used for both REST and GRPC versions of the Yandex.Cloud API.
-
-## Repository structure
-
-Repository structure reflects Yandex.Cloud service hierarchy. Folders represent different services of the platform and contain protobuf files. Each service folder contains resource data structure and interface definitions for API calls.
diff --git a/cloud/bitbucket/public-api/internal-CONTRIBUTING.md b/cloud/bitbucket/public-api/internal-CONTRIBUTING.md
deleted file mode 100644
index 50fabbd47e..0000000000
--- a/cloud/bitbucket/public-api/internal-CONTRIBUTING.md
+++ /dev/null
@@ -1,105 +0,0 @@
-## Public API development guide
-
-All cloud control plane API definition based on [GRPC](https://grpc.io).
-
-This repo contains:
-- public api definition
-- public api documentation
-
-Public API definitions used only for:
- - API Gateway
- - Public SDK
- - Docs generation
-
-For other purpose see [private API](https://bb.yandex-team.ru/projects/CLOUD/repos/private-api/browse)
-
-#### API definition style
-
-Layout `yandex/<service>/<version>/*.proto`
- - service in cloud product meaning, for example, compute, mds, microcosm, etc
- - version format for GA releases `v<N>`, for pre-GA releases `v<N><pre-release-name><M>`, ex: `v1tp1`
-
-For entity and service definitions use two .proto file:
- - `<entity>.proto`, use for entity structure definition
- - `<entity>_service.proto`, use for grpc service definition
-
-All methods for operations tracking and listing should be placed in `<entity>_service.proto`
-
-For the scalar fields of the message use one of the following types:
- - `int64`, everywhere for integer numbers. This allows not to think about overflow
- - `double`, everywhere for real numbers. This allows not to think about overflow
- - `string`, `bool`, no comments
-
-If you want to use any other scalar type, first take advice from API design team.
-
-See compute service API as a reference.
-
-## Setup repository
-
-### POSIX (maxOS or Linux)
-
-#### Prerequisites
-
-- make, curl
-- on macOS, protoc (should be >= 3.5) or Homebrew (protoc will be installed using brew)
-- on Linux, protoc (should be >= 3.5) or sudo access to install it from github
-
-#### Steps
-
-- clone this repo
-
-*Example:*
-```
-mkdir -p ~/cloud
-cd ~/cloud
-git clone https://bb.yandex-team.ru/scm/cloud/public-api.git
-cd ~/public-api
-```
-
-### Windows
-
-#### Prerequisites
-
-- Download and install [Git](https://git-scm.com/download/win) with mingw command line
-- Download and install [MinGW](https://sourceforge.net/projects/mingw/files/latest/download)
-- Configure MinGW: Run `C:\MinGW\bin\mingw-get install mingw32-base`
-
-
-#### Steps
-
-- Everything should be run under Git Bash (Start Menu -> Git Bash)
-- Make sure `mingw32-make` is in your `PATH`. As a backup, just run: `export PATH=$PATH:/c/MinGW/bin`
-
-- clone this repo
-
-*Example:*
-```
-mkdir -p ~/cloud
-cd ~/cloud
-git clone https://bb.yandex-team.ru/scm/cloud/public-api.git
-cd ~/public-api
-```
-
-
-## Working with repository
-*For Windows use `mingw32-make` instead of `make`*
-
-### Main commands
-- `make` to run lint and generate docs
- - Equivalent of running both `make lint` and `make generate`
- - This is run in every Pull Request Build. If `make` is not successful your PR will not be merged.
-- `make lint` to run lint
- - Run custom .proto files checks that forces API definition style.
- - To build proto tools from source set 'BUILD_TOOLS' variable to 1.
-- `make generate` to generate Open API files that used for docs generation.
- - Output will be in `${repo_root}/generated`
-
-
-### Utility commands
-These commands should not be required.
- - `make clean-generated` to delete all generated docs. In case any bugs happen (doc is not being regenerated), run this command and then run `make generate`
- - `make clean` to delete all binary dependencies (protobuf generator plugins, etc.) and generated docs.
-
-## Questions?
-Ask in [YC.API doc](https://t.me/joinchat/BLcQWA1_s-OlyJ_4NMhaEA) chat!
-
diff --git a/cloud/bitbucket/public-api/openapi-descriptions.yaml b/cloud/bitbucket/public-api/openapi-descriptions.yaml
deleted file mode 100644
index 23781c5516..0000000000
--- a/cloud/bitbucket/public-api/openapi-descriptions.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-replace-description:
- google.protobuf.Empty: |-
- Empty JSON object `` {} ``.
- google.protobuf.FieldMask: |-
- A comma-separated names off ALL fields to be updated.
- Оnly the specified fields will be changed. The others will be left untouched.
- If the field is specified in `` updateMask `` and no value for that field was sent in the request,
- the field's value will be reset to the default. The default value for most fields is null or 0.
-
- If `` updateMask `` is not sent in the request, all fields' values will be updated.
- Fields specified in the request will be updated to provided values.
- The rest of the fields will be reset to the default.
-
- google.rpc.Status: |-
- The error result of the operation in case of failure or cancellation.
- google.rpc.Status.code: |-
- Error code. An enum value of [google.rpc.Code][google.rpc.Code].
- google.rpc.Status.message: |-
- An error message.
- google.rpc.Status.details: |-
- A list of messages that carry the error details.
-
-append-format:
- google.protobuf.Timestamp: |-
- String in [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) text format.
diff --git a/cloud/bitbucket/public-api/yandex/cloud/CMakeLists.txt b/cloud/bitbucket/public-api/yandex/cloud/CMakeLists.txt
deleted file mode 100644
index 484f915bfd..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/CMakeLists.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(public-api-yandex-cloud)
-set_property(TARGET public-api-yandex-cloud PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET public-api-yandex-cloud PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/public-api
-)
-target_include_directories(public-api-yandex-cloud PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_include_directories(public-api-yandex-cloud PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_link_libraries(public-api-yandex-cloud PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- contrib-libs-protobuf
-)
-target_proto_messages(public-api-yandex-cloud PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/imports.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/validation.proto
-)
-target_proto_addincls(public-api-yandex-cloud
- ./cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(public-api-yandex-cloud
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_proto_plugin(public-api-yandex-cloud
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/public-api/yandex/cloud/access/CMakeLists.txt b/cloud/bitbucket/public-api/yandex/cloud/access/CMakeLists.txt
deleted file mode 100644
index f4b8bc8fb4..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/access/CMakeLists.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yandex-cloud-access)
-set_property(TARGET yandex-cloud-access PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET yandex-cloud-access PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/public-api
-)
-target_include_directories(yandex-cloud-access PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_include_directories(yandex-cloud-access PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_link_libraries(yandex-cloud-access PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- public-api-yandex-cloud
- contrib-libs-protobuf
-)
-target_proto_messages(yandex-cloud-access PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/access/access.proto
-)
-target_proto_addincls(yandex-cloud-access
- ./cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(yandex-cloud-access
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_proto_plugin(yandex-cloud-access
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/public-api/yandex/cloud/access/access.proto b/cloud/bitbucket/public-api/yandex/cloud/access/access.proto
deleted file mode 100644
index dd1191088d..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/access/access.proto
+++ /dev/null
@@ -1,120 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.access;
-
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/access;access";
-option java_package = "yandex.cloud.api.access";
-
-message Subject {
- // ID of the subject.
- //
- // It can contain one of the following values:
- // * `allAuthenticatedUsers`: A special system identifier that represents anyone
- // who is authenticated. It can be used only if the [type] is `system`.
- // * `allUsers`: A special system identifier that represents anyone. No authentication is required.
- // For example, you don't need to specify the IAM token in an API query.
- // * `<cloud generated id>`: An identifier that represents a user account.
- // It can be used only if the [type] is `userAccount`, `federatedUser` or `serviceAccount`.
- string id = 1 [(required) = true, (length) = "<=50"];
-
- // Type of the subject.
- //
- // It can contain one of the following values:
- // * `userAccount`: An account on Yandex or Yandex Connect, added to Yandex Cloud.
- // * `serviceAccount`: A service account. This type represents the [yandex.cloud.iam.v1.ServiceAccount] resource.
- // * `federatedUser`: A federated account. This type represents a user from an identity federation, like Active Directory.
- // * `system`: System group. This type represents several accounts with a common system identifier.
- //
- // For more information, see [Subject to which the role is assigned](/docs/iam/concepts/access-control/#subject).
- string type = 2 [(required) = true, (length) = "<=100"];
-}
-
-message AccessBinding {
- // ID of the [yandex.cloud.iam.v1.Role] that is assigned to the [subject].
- string role_id = 1 [(required) = true, (length) = "<=50"];
-
- // Identity for which access binding is being created.
- // It can represent an account with a unique ID or several accounts with a system identifier.
- Subject subject = 2 [(required) = true];
-}
-
-message ListAccessBindingsRequest {
- // ID of the resource to list access bindings for.
- //
- // To get the resource ID, use a corresponding List request.
- // For example, use the [yandex.cloud.resourcemanager.v1.CloudService.List] request to get the Cloud resource ID.
- string resource_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page that should be returned. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListAccessBindingsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "<=1000"];
-
- // Page token. Set [page_token]
- // to the [ListAccessBindingsResponse.next_page_token]
- // returned by a previous list request to get the next page of results.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessBindingsResponse {
- // List of access bindings for the specified resource.
- repeated AccessBinding access_bindings = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListAccessBindingsRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListAccessBindingsRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message SetAccessBindingsRequest {
- // ID of the resource for which access bindings are being set.
- //
- // To get the resource ID, use a corresponding List request.
- string resource_id = 1 [(required) = true, (length) = "<=50"];
-
- // Access bindings to be set. For more information, see [Access Bindings](/docs/iam/concepts/access-control/#access-bindings).
- repeated AccessBinding access_bindings = 2 [(required) = true];
-}
-
-message SetAccessBindingsMetadata {
- // ID of the resource for which access bindings are being set.
- string resource_id = 1;
-}
-
-message UpdateAccessBindingsRequest {
- // ID of the resource for which access bindings are being updated.
- string resource_id = 1 [(required) = true, (length) = "<=50"];
-
- // Updates to access bindings.
- repeated AccessBindingDelta access_binding_deltas = 2 [(size) = ">0", (required) = true];
-}
-
-message UpdateAccessBindingsMetadata {
- // ID of the resource for which access bindings are being updated.
- string resource_id = 1;
-}
-
-enum AccessBindingAction {
- ACCESS_BINDING_ACTION_UNSPECIFIED = 0;
-
- // Addition of an access binding.
- ADD = 1;
-
- // Removal of an access binding.
- REMOVE = 2;
-}
-
-message AccessBindingDelta {
- // The action that is being performed on an access binding.
- AccessBindingAction action = 1 [(required) = true];
-
- // Access binding. For more information, see [Access Bindings](/docs/iam/concepts/access-control/#access-bindings).
- AccessBinding access_binding = 2 [(required) = true];
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/CMakeLists.txt b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/CMakeLists.txt
deleted file mode 100644
index 34e3764b0e..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/CMakeLists.txt
+++ /dev/null
@@ -1,69 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cloud-iam-v1)
-set_property(TARGET cloud-iam-v1 PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET cloud-iam-v1 PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/public-api
-)
-target_include_directories(cloud-iam-v1 PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_include_directories(cloud-iam-v1 PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_link_libraries(cloud-iam-v1 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- yandex-cloud-api
- cloud-api-tools
- public-api-yandex-cloud
- yandex-cloud-access
- yandex-cloud-operation
- contrib-libs-protobuf
-)
-target_proto_messages(cloud-iam-v1 PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account_service.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/iam/v1/yandex_passport_user_account_service.proto
-)
-target_proto_addincls(cloud-iam-v1
- ./cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(cloud-iam-v1
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_proto_plugin(cloud-iam-v1
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key.proto
deleted file mode 100644
index 2fea0dee5c..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// An ApiKey resource. For more information, see [Api-Key](/docs/iam/concepts/authorization/api-key).
-message ApiKey {
- // ID of the API Key.
- string id = 1;
-
- // ID of the service account that the API key belongs to.
- string service_account_id = 2;
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 3;
-
- // Description of the API key. 0-256 characters long.
- string description = 4;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key_service.proto
deleted file mode 100644
index 502a26f210..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/api_key_service.proto
+++ /dev/null
@@ -1,167 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/iam/v1/api_key.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing API keys.
-service ApiKeyService {
- // Retrieves the list of API keys for the specified service account.
- rpc List (ListApiKeysRequest) returns (ListApiKeysResponse) {
- option (google.api.http) = {get: "/iam/v1/apiKeys"};
- }
-
- // Returns the specified API key.
- //
- // To get the list of available API keys, make a [List] request.
- rpc Get (GetApiKeyRequest) returns (ApiKey) {
- option (google.api.http) = {get: "/iam/v1/apiKeys/{api_key_id}"};
- }
-
- // Creates an API key for the specified service account.
- rpc Create (CreateApiKeyRequest) returns (CreateApiKeyResponse) {
- option (google.api.http) = {post: "/iam/v1/apiKeys" body: "*"};
- }
-
- // Updates the specified API key.
- rpc Update (UpdateApiKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {patch: "/iam/v1/apiKeys/{api_key_id}" body: "*"};
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateApiKeyMetadata"
- response: "ApiKey"
- };
- }
-
- // Deletes the specified API key.
- rpc Delete (DeleteApiKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {delete: "/iam/v1/apiKeys/{api_key_id}"};
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteApiKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Retrieves the list of operations for the specified API key.
- rpc ListOperations (ListApiKeyOperationsRequest) returns (ListApiKeyOperationsResponse) {
- option (google.api.http) = {get: "/iam/v1/apiKeys/{api_key_id}/operations"};
- }
-}
-
-message GetApiKeyRequest {
- // ID of the API key to return.
- // To get the API key ID, use a [ApiKeyService.List] request.
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListApiKeysRequest {
- // ID of the service account to list API keys for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListApiKeysResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListApiKeysResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListApiKeysResponse {
- // List of API keys.
- repeated ApiKey api_keys = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListApiKeysRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListApiKeysRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateApiKeyRequest {
- // ID of the service account to create an API key for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
-
- // Description of the API key.
- string description = 2 [(length) = "<=256"];
-}
-
-message CreateApiKeyResponse {
- // ApiKey resource.
- ApiKey api_key = 1;
-
- // Secret part of the API key. This secret key you may use in the requests for authentication.
- string secret = 2;
-}
-
-message UpdateApiKeyRequest {
- // ID of the ApiKey resource to update.
- // To get the API key ID, use a [ApiKeyService.List] request.
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-
- // Field mask that specifies which fields of the ApiKey resource are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Description of the API key.
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateApiKeyMetadata {
- // ID of the ApiKey resource that is being updated.
- string api_key_id = 1;
-}
-
-message DeleteApiKeyRequest {
- // ID of the API key to delete.
- // To get the API key ID, use a [ApiKeyService.List] request.
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteApiKeyMetadata {
- // ID of the API key that is being deleted.
- string api_key_id = 1;
-}
-
-message ListApiKeyOperationsRequest {
- // ID of the key to list operations for.
- string api_key_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListApiKeyOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token] to the
- // [ListApiKeyOperationsResponse.next_page_token] returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListApiKeyOperationsResponse {
- // List of operations for the specified API key.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListApiKeyOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListApiKeyOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key.proto
deleted file mode 100644
index 09ac7ed57f..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.awscompatibility;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility;awscompatibility";
-option java_package = "yandex.cloud.api.iam.v1.awscompatibility";
-
-// An access key.
-// For more information, see [AWS-compatible access keys](/docs/iam/concepts/authorization/access-key).
-message AccessKey {
- // ID of the AccessKey resource.
- // It is used to manage secret credentials: an access key ID and a secret access key.
- string id = 1;
-
- // ID of the service account that the access key belongs to.
- string service_account_id = 2;
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 3;
-
- // Description of the access key. 0-256 characters long.
- string description = 4;
-
- // ID of the access key.
- // The key is AWS compatible.
- string key_id = 5;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key_service.proto
deleted file mode 100644
index 3e1715d4a3..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/access_key_service.proto
+++ /dev/null
@@ -1,168 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.awscompatibility;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/iam/v1/awscompatibility/access_key.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility;awscompatibility";
-option java_package = "yandex.cloud.api.iam.v1.awscompatibility";
-
-// A set of methods for managing access keys.
-service AccessKeyService {
- // Retrieves the list of access keys for the specified service account.
- rpc List (ListAccessKeysRequest) returns (ListAccessKeysResponse) {
- option (google.api.http) = {get: "/iam/aws-compatibility/v1/accessKeys"};
- }
-
- // Returns the specified access key.
- //
- // To get the list of available access keys, make a [List] request.
- rpc Get (GetAccessKeyRequest) returns (AccessKey) {
- option (google.api.http) = {get: "/iam/aws-compatibility/v1/accessKeys/{access_key_id}"};
- }
-
- // Creates an access key for the specified service account.
- rpc Create (CreateAccessKeyRequest) returns (CreateAccessKeyResponse) {
- option (google.api.http) = {post: "/iam/aws-compatibility/v1/accessKeys" body: "*"};
- }
-
- // Updates the specified access key.
- rpc Update (UpdateAccessKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {patch: "/iam/aws-compatibility/v1/accessKeys/{access_key_id}" body: "*"};
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateAccessKeyMetadata"
- response: "AccessKey"
- };
- }
-
- // Deletes the specified access key.
- rpc Delete (DeleteAccessKeyRequest) returns (operation.Operation) {
- option (google.api.http) = {delete: "/iam/aws-compatibility/v1/accessKeys/{access_key_id}"};
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteAccessKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Retrieves the list of operations for the specified access key.
- rpc ListOperations (ListAccessKeyOperationsRequest) returns (ListAccessKeyOperationsResponse) {
- option (google.api.http) = {get: "/iam/aws-compatibility/v1/accessKeys/{access_key_id}/operations"};
- }
-}
-
-message GetAccessKeyRequest {
- // ID of the AccessKey resource to return.
- // To get the access key ID, use a [AccessKeyService.List] request.
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListAccessKeysRequest {
- // ID of the service account to list access keys for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListAccessKeysResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "<=1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListAccessKeysResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessKeysResponse {
- // List of access keys.
- repeated AccessKey access_keys = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListAccessKeysRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListAccessKeysRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateAccessKeyRequest {
- // ID of the service account to create an access key for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 1 [(length) = "<=50"]; // use current subject identity if this not set
-
- // Description of the access key.
- string description = 2 [(length) = "<=256"];
-}
-
-message CreateAccessKeyResponse {
- // AccessKey resource.
- AccessKey access_key = 1;
-
- // Secret access key.
- // The key is AWS compatible.
- string secret = 2;
-}
-
-message UpdateAccessKeyRequest {
- // ID of the AccessKey resource to update.
- // To get the access key ID, use a [AccessKeyService.List] request.
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-
- // Field mask that specifies which fields of the Accesskey resource are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Description of the access key.
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateAccessKeyMetadata {
- // ID of the AccessKey resource that is being updated.
- string access_key_id = 1;
-}
-
-message DeleteAccessKeyRequest {
- // ID of the access key to delete.
- // To get the access key ID, use a [AccessKeyService.List] request.
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteAccessKeyMetadata {
- // ID of the access key that is being deleted.
- string access_key_id = 1;
-}
-
-message ListAccessKeyOperationsRequest {
- // ID of the key to list operations for.
- string access_key_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListAccessKeyOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token] to the
- // [ListAccessKeyOperationsResponse.next_page_token] returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListAccessKeyOperationsResponse {
- // List of operations for the specified access key.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListAccessKeyOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListAccessKeyOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/openapi-meta.yaml b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/openapi-meta.yaml
deleted file mode 100644
index 1cf3d3f664..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/awscompatibility/openapi-meta.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-info:
- title: Yandex IAM API
- version: v1
- description: This API reference is organized by resource. Actions are performed by sending HTTP requests to resource URLs or making RPC calls. For more information about Yandex.Cloud API architecture, see [API Concepts](/docs/api-design-guide/).
- x-y-docUri: api-ref
- x-y-baseUrl: "https://iam.api.cloud.yandex.net"
-docName: "/docs/iam/api-ref"
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.proto
deleted file mode 100644
index 034e84ec0e..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.proto
+++ /dev/null
@@ -1,52 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing IAM tokens.
-service IamTokenService {
- // Creates an IAM token for the specified identity.
- rpc Create (CreateIamTokenRequest) returns (CreateIamTokenResponse) {
- option (google.api.http) = {post: "/iam/v1/tokens" body: "*"};
- }
-
- // Create iam token for service account.
- rpc CreateForServiceAccount (CreateIamTokenForServiceAccountRequest) returns (CreateIamTokenResponse) {
- option (google.api.http) = {post: "/iam/v1/tokens:createForServiceAccount" body: "*"};
- };
-}
-
-message CreateIamTokenRequest {
- oneof identity {
- option (exactly_one) = true;
-
- // OAuth token for a Yandex account.
- // For more information, see [OAuth token](/docs/iam/concepts/authorization/oauth-token).
- string yandex_passport_oauth_token = 1;
-
- // JSON Web Token (JWT) for a service account.
- // For more information, see [Get IAM token for a service account](/docs/iam/operations/iam-token/create-for-sa).
- string jwt = 2;
- }
-}
-
-message CreateIamTokenResponse {
- // IAM token for the specified identity.
- //
- // You should pass the token in the `Authorization` header for any further API requests.
- // For example, `Authorization: Bearer [iam_token]`.
- string iam_token = 1;
-
- // IAM token expiration time.
- google.protobuf.Timestamp expires_at = 2;
-}
-
-message CreateIamTokenForServiceAccountRequest {
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key.proto
deleted file mode 100644
index 22eff7180e..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key.proto
+++ /dev/null
@@ -1,44 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A Key resource. For more information, see [Authorized keys](/docs/iam/concepts/authorization/key).
-message Key {
- enum Algorithm {
- ALGORITHM_UNSPECIFIED = 0;
-
- // RSA with a 2048-bit key size. Default value.
- RSA_2048 = 1;
-
- // RSA with a 4096-bit key size.
- RSA_4096 = 2;
- }
-
- // ID of the Key resource.
- string id = 1;
-
- oneof subject {
- // ID of the user account that the Key resource belongs to.
- string user_account_id = 2;
-
- // ID of the service account that the Key resource belongs to.
- string service_account_id = 3;
- }
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 4;
-
- // Description of the Key resource. 0-256 characters long.
- string description = 5;
-
- // An algorithm used to generate a key pair of the Key resource.
- Algorithm key_algorithm = 6;
-
- // A public key of the Key resource.
- string public_key = 7;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key_service.proto
deleted file mode 100644
index aec6ecedfb..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/key_service.proto
+++ /dev/null
@@ -1,187 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/iam/v1/key.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing Key resources.
-service KeyService {
- // Returns the specified Key resource.
- //
- // To get the list of available Key resources, make a [List] request.
- rpc Get (GetKeyRequest) returns (Key) {
- option (google.api.http) = { get: "/iam/v1/keys/{key_id}" };
- }
-
- // Retrieves the list of Key resources for the specified service account.
- rpc List (ListKeysRequest) returns (ListKeysResponse) {
- option (google.api.http) = { get: "/iam/v1/keys" };
- }
-
- // Creates a key pair for the specified service account.
- rpc Create (CreateKeyRequest) returns (CreateKeyResponse) {
- option (google.api.http) = { post: "/iam/v1/keys" body: "*" };
- }
-
- // Updates the specified key pair.
- rpc Update (UpdateKeyRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/keys/{key_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateKeyMetadata"
- response: "Key"
- };
- }
-
- // Deletes the specified key pair.
- rpc Delete (DeleteKeyRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/keys/{key_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteKeyMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Lists operations for the specified key.
- rpc ListOperations (ListKeyOperationsRequest) returns (ListKeyOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/keys/{key_id}/operations" };
- }
-}
-
-message GetKeyRequest {
- // ID of the Key resource to return.
- // To get the ID use a [KeyService.List] request.
- string key_id = 1 [(required) = true, (length) = "<=50"];
-
- // Output format of the key.
- KeyFormat format = 2;
-}
-
-message ListKeysRequest {
- // Output format of the key.
- KeyFormat format = 1;
-
- // ID of the service account to list key pairs for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 2 [(length) = "<=50"]; // use userAccount identity if this not set
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListKeysResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 3 [(value) = "<=1000"];
-
- // Page token. To get the next page of results, set [page_token] to the
- // [ListKeysResponse.next_page_token] returned by a previous list request.
- string page_token = 4 [(length) = "<=100"];
-}
-
-message ListKeysResponse {
- // List of Key resources.
- repeated Key keys = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListKeysRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListKeysRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateKeyRequest {
- // ID of the service account to create a key pair for.
- // To get the service account ID, use a [yandex.cloud.iam.v1.ServiceAccountService.List] request.
- // If not specified, it defaults to the subject that made the request.
- string service_account_id = 1 [(length) = "<=50"]; // use userAccount identity if this not set
-
- // Description of the key pair.
- string description = 2 [(length) = "<=256"];
-
- // Output format of the key.
- KeyFormat format = 3;
-
- // An algorithm used to generate a key pair of the Key resource.
- Key.Algorithm key_algorithm = 4;
-}
-
-message CreateKeyResponse {
- // Key resource.
- Key key = 1;
-
- // A private key of the Key resource.
- // This key must be stored securely.
- string private_key = 2;
-}
-
-message UpdateKeyRequest {
- // ID of the Key resource to update.
- // To get key pair ID, use a [KeyService.List] request.
- string key_id = 1 [(required) = true, (length) = "<=50"];
-
- // Field mask that specifies which fields of the Key resource are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Description of the key pair.
- string description = 3 [(length) = "<=256"];
-}
-
-message UpdateKeyMetadata {
- // ID of the Key resource that is being updated.
- string key_id = 1;
-}
-
-message DeleteKeyRequest {
- // ID of the key to delete.
- // To get key ID use a [KeyService.List] request.
- string key_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteKeyMetadata {
- // ID of the key that is being deleted.
- string key_id = 1;
-}
-
-message ListKeyOperationsRequest {
- // ID of the key to list operations for.
- string key_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListKeyOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token] to the
- // [ListKeyOperationsResponse.next_page_token] returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListKeyOperationsResponse {
- // List of operations for the specified key.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListKeyOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListKeyOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-enum KeyFormat {
- option (cloud.api.tools.enumeration).lint_skip.unspecified_value = true;
-
- // Privacy-Enhanced Mail (PEM) format. Default value.
- PEM_FILE = 0;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/openapi-meta.yaml b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/openapi-meta.yaml
deleted file mode 100644
index 1cf3d3f664..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/openapi-meta.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-info:
- title: Yandex IAM API
- version: v1
- description: This API reference is organized by resource. Actions are performed by sending HTTP requests to resource URLs or making RPC calls. For more information about Yandex.Cloud API architecture, see [API Concepts](/docs/api-design-guide/).
- x-y-docUri: api-ref
- x-y-baseUrl: "https://iam.api.cloud.yandex.net"
-docName: "/docs/iam/api-ref"
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role.proto
deleted file mode 100644
index ebbd02306f..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A Role resource. For more information, see [Roles](/docs/iam/concepts/access-control/roles).
-message Role {
- // ID of the role.
- string id = 1;
-
- // Description of the role. 0-256 characters long.
- string description = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role_service.proto
deleted file mode 100644
index bf20a1f94d..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/role_service.proto
+++ /dev/null
@@ -1,61 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/validation.proto";
-import "yandex/cloud/iam/v1/role.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing Role resources.
-service RoleService {
- // Returns the specified Role resource.
- //
- // To get the list of available Role resources, make a [List] request.
- rpc Get (GetRoleRequest) returns (Role) {
- option (google.api.http) = {get: "/iam/v1/roles/{role_id}"};
- }
-
- // Retrieves the list of Role resources.
- rpc List (ListRolesRequest) returns (ListRolesResponse) {
- option (google.api.http) = {get: "/iam/v1/roles"};
- }
-}
-
-message GetRoleRequest {
- // ID of the Role resource to return.
- // To get the role ID, use a [RoleService.List] request.
- string role_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListRolesRequest {
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListRolesResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 1 [(value) = "<=1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListRolesResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 2 [(length) = "<=100"];
-
- // A filter expression that filters resources listed in the response.
- string filter = 3 [(length) = "<=1000"];
-}
-
-message ListRolesResponse {
- // List of Role resources.
- repeated Role roles = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListRolesRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListRolesRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate.proto
deleted file mode 100644
index ed7f14e68a..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.saml;
-
-import "yandex/cloud/validation.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml;saml";
-option java_package = "yandex.cloud.api.iam.v1.saml";
-
-// A certificate.
-message Certificate {
-
- // ID of the certificate.
- string id = 1 [(required) = true, (length) = "<=50"];
-
- // ID of the federation that the certificate belongs to.
- string federation_id = 2 [(required) = true, (length) = "<=50"];
-
- // Name of the certificate.
- string name = 3 [ (pattern) = "|[a-z][-a-z0-9]{1,61}[a-z0-9]"];
-
- // Description of the certificate.
- string description = 4 [(length) = "<=256"];
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 5;
-
- // Certificate data in PEM format.
- string data = 6 [(required) = true, (length) = "<=32000"];
-
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate_service.proto
deleted file mode 100644
index 99c7f2423b..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/certificate_service.proto
+++ /dev/null
@@ -1,186 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.saml;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/iam/v1/saml/certificate.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml;saml";
-option java_package = "yandex.cloud.api.iam.v1.saml";
-
-// A set of methods for managing certificates.
-service CertificateService {
- // Returns the specified certificate.
- //
- // To get the list of available certificates, make a [List] request.
- rpc Get (GetCertificateRequest) returns (Certificate) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates/{certificate_id}" };
- }
-
- // Retrieves the list of certificates in the specified federation.
- rpc List (ListCertificatesRequest) returns (ListCertificatesResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates" };
- }
-
- // Creates a certificate in the specified federation.
- rpc Create (CreateCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/certificates" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateCertificateMetadata"
- response: "Certificate"
- };
- }
-
- // Updates the specified certificate.
- rpc Update (UpdateCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/saml/certificates/{certificate_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateCertificateMetadata"
- response: "Certificate"
- };
- }
-
- // Deletes the specified certificate.
- rpc Delete (DeleteCertificateRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/saml/certificates/{certificate_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteCertificateMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Lists operations for the specified certificate.
- rpc ListOperations (ListCertificateOperationsRequest) returns (ListCertificateOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/certificates/{certificate_id}/operations" };
- }
-}
-
-message GetCertificateRequest {
- // ID of the certificate to return.
- // To get the certificate ID, make a [CertificateService.List] request.
- string certificate_id = 1 [(length) = "<=50"];
-}
-
-message ListCertificatesRequest {
- // ID of the federation to list certificates in.
- // To get the federation ID make a [yandex.cloud.iam.v1.saml.FederationService.List] request.
- string federation_id = 1 [(length) = "<=50", (required) = true];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size], the service returns a [ListCertificatesResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListCertificatesResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-
- // A filter expression that filters resources listed in the response.
- // The expression must specify:
- // 1. The field name. Currently you can use filtering only on [Certificate.name] field.
- // 2. An `=` operator.
- // 3. The value in double quotes (`"`). Must be 3-63 characters long and match the regular expression `[a-z][-a-z0-9]{1,61}[a-z0-9]`.
- string filter = 4 [(length) = "<=1000"];
-}
-
-message ListCertificatesResponse {
- // List of certificates.
- repeated Certificate certificates = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListCertificatesRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListCertificatesRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateCertificateRequest {
- // ID of the federation to add new certificate.
- // To get the federation ID make a [yandex.cloud.iam.v1.saml.FederationService.List] request.
- string federation_id = 1 [(length) = "<=50"];
-
- // Name of the certificate.
- // The name must be unique within the federation.
- string name = 2 [(pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
-
- // Description of the certificate.
- string description = 3 [(length) = "<=256"];
-
- // Certificate data in PEM format.
- string data = 4 [(length) = "<=32000"];
-}
-
-message CreateCertificateMetadata {
- // ID of the certificate that is being created.
- string certificate_id = 1;
-}
-
-message UpdateCertificateRequest {
- // ID of the certificate to update.
- // To get the certificate ID, make a [CertificateService.List] request.
- string certificate_id = 1 [(length) = "<=50"];
-
- // Field mask that specifies which fields of the certificate are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Name of the certificate.
- // The name must be unique within the federation.
- string name = 3 [(pattern) = "|[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
-
- // Description of the certificate.
- string description = 4 [(length) = "<=256"];
-
- // Certificate data in PEM format.
- string data = 5 [(length) = "<=32000"];
-}
-
-message UpdateCertificateMetadata {
- // ID of the certificate that is being updated.
- string certificate_id = 1;
-}
-
-message DeleteCertificateRequest {
- // ID of the certificate to delete.
- // To get the certificate ID, make a [CertificateService.List] request.
- string certificate_id = 1 [(length) = "<=50"];
-}
-
-message DeleteCertificateMetadata {
- // ID of the certificate that is being deleted.
- string certificate_id = 1;
-}
-
-message ListCertificateOperationsRequest {
- // ID of the certificate to list operations for.
- string certificate_id = 1 [(length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size], the service returns a [ListCertificateOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListCertificateOperationsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListCertificateOperationsResponse {
- // List of operations for the specified certificate.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListCertificateOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListCertificateOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation.proto
deleted file mode 100644
index 71c8e1f9b6..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation.proto
+++ /dev/null
@@ -1,83 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.saml;
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml;saml";
-option java_package = "yandex.cloud.api.iam.v1.saml";
-
-// A federation.
-// For more information, see [SAML-compatible identity federations](/docs/iam/concepts/users/identity-federations).
-message Federation {
-
- // ID of the federation.
- string id = 1 [(required) = true, (length) = "<=50"];
-
- // ID of the folder that the federation belongs to.
- string folder_id = 2 [(required) = true, (length) = "<=50"];
-
- // Name of the federation.
- string name = 3 [(required) = true, (pattern) = "|[a-z][-a-z0-9]{1,61}[a-z0-9]"];
-
- // Description of the federation.
- string description = 4 [(length) = "<=256"];
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 5;
-
- // Browser cookie lifetime in seconds.
- // If the cookie is still valid, the management console
- // authenticates the user immediately and redirects them to the home page.
- google.protobuf.Duration cookie_max_age = 6 [(value) = "10m-12h"];
-
- // Add new users automatically on successful authentication.
- // The user will get the `resource-manager.clouds.member` role automatically,
- // but you need to grant other roles to them.
- //
- // If the value is `false`, users who aren't added to the cloud
- // can't log in, even if they have authenticated on your server.
- bool auto_create_account_on_login = 7;
-
- // ID of the IdP server to be used for authentication.
- // The IdP server also responds to IAM with this ID after the user authenticates.
- string issuer = 8 [(required) = true, (length) = "<=8000"];
-
- // Single sign-on endpoint binding type. Most Identity Providers support the `POST` binding type.
- //
- // SAML Binding is a mapping of a SAML protocol message onto standard messaging
- // formats and/or communications protocols.
- BindingType sso_binding = 9;
-
- // Single sign-on endpoint URL.
- // Specify the link to the IdP login page here.
- string sso_url = 10 [(required) = true, (length) = "<=8000"];
-
- // Federation security settings.
- FederationSecuritySettings security_settings = 11;
-
- // Use case insensitive Name IDs.
- bool case_insensitive_name_ids = 12;
-}
-
-enum BindingType {
- BINDING_TYPE_UNSPECIFIED = 0;
-
- // HTTP POST binding.
- POST = 1;
-
- // HTTP redirect binding.
- REDIRECT = 2;
-
- // HTTP artifact binding.
- ARTIFACT = 3;
-
-}
-
-// Federation security settings.
-message FederationSecuritySettings {
- // Enable encrypted assertions.
- bool encrypted_assertions = 1;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation_service.proto
deleted file mode 100644
index 363431aaf4..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/federation_service.proto
+++ /dev/null
@@ -1,317 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1.saml;
-
-import "google/api/annotations.proto";
-import "google/protobuf/duration.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/iam/v1/user_account.proto";
-import "yandex/cloud/iam/v1/saml/federation.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml;saml";
-option java_package = "yandex.cloud.api.iam.v1.saml";
-
-// A set of methods for managing federations.
-service FederationService {
- // Returns the specified federation.
- //
- // To get the list of available federations, make a [List] request.
- rpc Get (GetFederationRequest) returns (Federation) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}" };
- }
-
- // Retrieves the list of federations in the specified folder.
- rpc List (ListFederationsRequest) returns (ListFederationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations" };
- }
-
- // Creates a federation in the specified folder.
- rpc Create (CreateFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/federations" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateFederationMetadata"
- response: "Federation"
- };
- }
-
- // Updates the specified federation.
- rpc Update (UpdateFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/saml/federations/{federation_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateFederationMetadata"
- response: "Federation"
- };
- }
-
- // Deletes the specified federation.
- rpc Delete (DeleteFederationRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/saml/federations/{federation_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteFederationMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Adds users to the specified federation.
- rpc AddUserAccounts (AddFederatedUserAccountsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/saml/federations/{federation_id}:addUserAccounts" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "AddFederatedUserAccountsMetadata"
- response: "AddFederatedUserAccountsResponse"
- };
- }
-
- // Lists users for the specified federation.
- rpc ListUserAccounts (ListFederatedUserAccountsRequest) returns (ListFederatedUserAccountsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}:listUserAccounts" };
- }
-
- // Lists operations for the specified federation.
- rpc ListOperations (ListFederationOperationsRequest) returns (ListFederationOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/saml/federations/{federation_id}/operations" };
- }
-
-}
-
-message GetFederationRequest {
- // ID of the federation to return.
- // To get the federation ID, make a [FederationService.List] request.
- string federation_id = 1 [(length) = "<=50"];
-}
-
-message ListFederationsRequest {
- oneof scope {
- // ID of the cloud to list federations in.
- // To get the cloud ID, make a [yandex.cloud.resourcemanager.v1.CloudService.List] request.
- string cloud_id = 1 [(length) = "<=50"];
-
- // ID of the folder to list federations in.
- // To get the folder ID, make a [yandex.cloud.resourcemanager.v1.FolderService.List] request.
- string folder_id = 2 [(length) = "<=50"];
- }
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListFederationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100
- int64 page_size = 3 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListFederationsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 4 [(length) = "<=50"];
-
- // A filter expression that filters resources listed in the response.
- // The expression must specify:
- // 1. The field name. Currently you can use filtering only on the [Federation.name] field.
- // 2. An `=` operator.
- // 3. The value in double quotes (`"`). Must be 3-63 characters long and match the regular expression `[a-z][-a-z0-9]{1,61}[a-z0-9]`.
- string filter = 5 [(length) = "<=1000"];
-}
-
-message ListFederationsResponse {
- // List of federations.
- repeated Federation federations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListFederationsRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListFederationsRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateFederationRequest {
- // ID of the folder to create a federation in.
- // To get the folder ID, make a [yandex.cloud.resourcemanager.v1.FolderService.List] request.
- string folder_id = 1 [(length) = "<=50"];
-
- // Name of the federation.
- // The name must be unique within the cloud.
- string name = 2 [(pattern) = "[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
-
- // Description of the federation.
- string description = 3 [(length) = "<=256"];
-
- // Browser cookie lifetime in seconds.
- // If the cookie is still valid, the management console
- // authenticates the user immediately and redirects them to the home page.
- // The default value is `8h`.
- google.protobuf.Duration cookie_max_age = 4 [(value) = "10m-12h"];
-
- // Add new users automatically on successful authentication.
- // The user will get the `resource-manager.clouds.member` role automatically,
- // but you need to grant other roles to them.
- //
- // If the value is `false`, users who aren't added to the cloud
- // can't log in, even if they have authenticated on your server.
- bool auto_create_account_on_login = 5;
-
- // ID of the IdP server to be used for authentication.
- // The IdP server also responds to IAM with this ID after the user authenticates.
- string issuer = 6 [(required) = true, (length) = "<=8000"];
-
- // Single sign-on endpoint binding type. Most Identity Providers support the `POST` binding type.
- //
- // SAML Binding is a mapping of a SAML protocol message onto standard messaging
- // formats and/or communications protocols.
- BindingType sso_binding = 7;
-
- // Single sign-on endpoint URL.
- // Specify the link to the IdP login page here.
- string sso_url = 8 [(required) = true, (length) = "<=8000"];
-
- // Federation security settings.
- FederationSecuritySettings security_settings = 9;
-
- // Use case insensitive Name IDs.
- bool case_insensitive_name_ids = 10;
-}
-
-message CreateFederationMetadata {
- // ID of the federation that is being created.
- string federation_id = 1;
-}
-
-message UpdateFederationRequest {
- // ID of the federation to update.
- // To get the federation ID, make a [FederationService.List] request.
- string federation_id = 1 [(length) = "<=50"];
-
- // Field mask that specifies which fields of the federation are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Name of the federation.
- // The name must be unique within the cloud.
- string name = 3 [(pattern) = "|[a-z]([-a-z0-9]{0,61}[a-z0-9])?"];
-
- // Description of the federation.
- string description = 4 [(length) = "<=256"];
-
- // Browser cookie lifetime in seconds.
- // If the cookie is still valid, the management console
- // authenticates the user immediately and redirects them to the home page.
- // The default value is `8h`.
- google.protobuf.Duration cookie_max_age = 5 [(value) = "10m-12h"];
-
- // Add new users automatically on successful authentication.
- // The user will get the `resource-manager.clouds.member` role automatically,
- // but you need to grant other roles to them.
- //
- // If the value is `false`, users who aren't added to the cloud
- // can't log in, even if they have authenticated on your server.
- bool auto_create_account_on_login = 6;
-
- // ID of the IdP server to be used for authentication.
- // The IdP server also responds to IAM with this ID after the user authenticates.
- string issuer = 7 [(required) = true, (length) = "<=8000"];
-
- // Single sign-on endpoint binding type. Most Identity Providers support the `POST` binding type.
- //
- // SAML Binding is a mapping of a SAML protocol message onto standard messaging
- // formats and/or communications protocols.
- BindingType sso_binding = 8;
-
- // Single sign-on endpoint URL.
- // Specify the link to the IdP login page here.
- string sso_url = 9 [(required) = true, (length) = "<=8000"];
-
- // Federation security settings.
- FederationSecuritySettings security_settings = 10;
-
- // Use case insensitive name ids.
- bool case_insensitive_name_ids = 12;
-}
-
-message UpdateFederationMetadata {
- // ID of the federation that is being updated.
- string federation_id = 1;
-}
-
-message DeleteFederationRequest {
- // ID of the federation to delete.
- // To get the federation ID, make a [FederationService.List] request.
- string federation_id = 1 [(length) = "<=50"];
-}
-
-message DeleteFederationMetadata {
- // ID of the federation that is being deleted.
- string federation_id = 1;
-}
-
-message AddFederatedUserAccountsRequest {
- // ID of the federation to add users.
- string federation_id = 1 [(length) = "<=50"];
- // Name IDs returned by the Identity Provider (IdP) on successful authentication.
- // These may be UPNs or user email addresses.
- repeated string name_ids = 2 [(length) = "<=1000"];
-}
-
-message AddFederatedUserAccountsMetadata {
- // ID of the federation that is being altered.
- string federation_id = 1;
-}
-
-message AddFederatedUserAccountsResponse {
- // List of users created by [FederationService.AddUserAccounts] request.
- repeated UserAccount user_accounts = 1;
-}
-
-message ListFederatedUserAccountsRequest {
- // ID of the federation to list user accounts for.
- string federation_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size], the service returns a [ListFederatedUserAccountsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListFederatedUserAccountsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListFederatedUserAccountsResponse {
- // List of user accounts for the specified federation.
- repeated UserAccount user_accounts = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListFederatedUserAccountsRequest.page_size], use the [next_page_token] as the value
- // for the [ListFederatedUserAccountsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message ListFederationOperationsRequest {
- // ID of the federation to list operations for.
- string federation_id = 1 [(length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size], the service returns a [ListFederationOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "0-1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListFederationOperationsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListFederationOperationsResponse {
- // List of operations for the specified federation.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListFederationOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListFederationOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/openapi-meta.yaml b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/openapi-meta.yaml
deleted file mode 100644
index f9d6775355..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/saml/openapi-meta.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-info:
- title: Yandex IAM SAML API
- version: v1
- description: This API reference is organized by resource. Actions are performed by sending HTTP requests to resource URLs or making RPC calls. For more information about Yandex.Cloud API architecture, see [API Concepts](/docs/api-design-guide/).
- x-y-docUri: api-ref
- x-y-baseUrl: "https://iam.api.cloud.yandex.net"
-docName: "/docs/iam/api-ref"
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account.proto
deleted file mode 100644
index c21e68a00f..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account.proto
+++ /dev/null
@@ -1,30 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/protobuf/timestamp.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A ServiceAccount resource. For more information, see [Service accounts](/docs/iam/concepts/users/service-accounts).
-message ServiceAccount {
- // ID of the service account.
- string id = 1;
-
- // ID of the folder that the service account belongs to.
- string folder_id = 2;
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 3;
-
- // Name of the service account.
- // The name is unique within the cloud. 3-63 characters long.
- string name = 4;
-
- // Description of the service account. 0-256 characters long.
- string description = 5;
-
- // Resource labels as `` key:value `` pairs. Maximum of 64 per resource.
- map<string, string> labels = 6;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account_service.proto
deleted file mode 100644
index 42f8576632..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/service_account_service.proto
+++ /dev/null
@@ -1,215 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "google/protobuf/field_mask.proto";
-import "yandex/cloud/api/operation.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/iam/v1/service_account.proto";
-import "yandex/cloud/access/access.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing ServiceAccount resources.
-service ServiceAccountService {
- // Returns the specified ServiceAccount resource.
- //
- // To get the list of available ServiceAccount resources, make a [List] request.
- rpc Get (GetServiceAccountRequest) returns (ServiceAccount) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{service_account_id}" };
- }
-
- // Retrieves the list of ServiceAccount resources in the specified folder.
- rpc List (ListServiceAccountsRequest) returns (ListServiceAccountsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts" };
- }
-
- // Creates a service account in the specified folder.
- rpc Create (CreateServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "CreateServiceAccountMetadata"
- response: "ServiceAccount"
- };
- }
-
- // Updates the specified service account.
- rpc Update (UpdateServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { patch: "/iam/v1/serviceAccounts/{service_account_id}" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "UpdateServiceAccountMetadata"
- response: "ServiceAccount"
- };
- }
-
- // Deletes the specified service account.
- rpc Delete (DeleteServiceAccountRequest) returns (operation.Operation) {
- option (google.api.http) = { delete: "/iam/v1/serviceAccounts/{service_account_id}" };
- option (yandex.cloud.api.operation) = {
- metadata: "DeleteServiceAccountMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- //access
-
- // Lists access bindings for the specified service account.
- rpc ListAccessBindings (access.ListAccessBindingsRequest) returns (access.ListAccessBindingsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{resource_id}:listAccessBindings" };
- }
-
- // Sets access bindings for the service account.
- rpc SetAccessBindings (access.SetAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts/{resource_id}:setAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "access.SetAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- }
-
- // Updates access bindings for the specified service account.
- rpc UpdateAccessBindings (access.UpdateAccessBindingsRequest) returns (operation.Operation) {
- option (google.api.http) = { post: "/iam/v1/serviceAccounts/{resource_id}:updateAccessBindings" body: "*" };
- option (yandex.cloud.api.operation) = {
- metadata: "access.UpdateAccessBindingsMetadata"
- response: "google.protobuf.Empty"
- };
- option (yandex.cloud.api.tools.method).lint_skip.http_verb = true;
- }
-
- // Lists operations for the specified service account.
- rpc ListOperations (ListServiceAccountOperationsRequest) returns (ListServiceAccountOperationsResponse) {
- option (google.api.http) = { get: "/iam/v1/serviceAccounts/{service_account_id}/operations" };
- }
-}
-
-message GetServiceAccountRequest {
- // ID of the ServiceAccount resource to return.
- // To get the service account ID, use a [ServiceAccountService.List] request.
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message ListServiceAccountsRequest {
- // ID of the folder to list service accounts in.
- // To get the folder ID, use a [yandex.cloud.resourcemanager.v1.FolderService.List] request.
- string folder_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size],
- // the service returns a [ListServiceAccountsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100
- int64 page_size = 2 [(value) = "<=1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListServiceAccountsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-
- // A filter expression that filters resources listed in the response.
- // The expression must specify:
- // 1. The field name. Currently you can use filtering only on the [ServiceAccount.name] field.
- // 2. An `=` operator.
- // 3. The value in double quotes (`"`). Must be 3-63 characters long and match the regular expression `[a-z][-a-z0-9]{1,61}[a-z0-9]`.
- string filter = 4 [(length) = "<=1000"];
-}
-
-message ListServiceAccountsResponse {
- // List of ServiceAccount resources.
- repeated ServiceAccount service_accounts = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListServiceAccountsRequest.page_size], use
- // the [next_page_token] as the value
- // for the [ListServiceAccountsRequest.page_token] query parameter
- // in the next list request. Each subsequent list request will have its own
- // [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
-
-message CreateServiceAccountRequest {
- // ID of the folder to create a service account in.
- // To get the folder ID, use a [yandex.cloud.resourcemanager.v1.FolderService.List] request.
- string folder_id = 1 [(required) = true, (length) = "<=50"];
-
- // Name of the service account.
- // The name must be unique within the cloud.
- string name = 2 [(required) = true, (pattern) = "|[a-z][-a-z0-9]{1,61}[a-z0-9]"];
-
- // Description of the service account.
- string description = 3 [(length) = "<=256"];
-
- // Resource labels as `` key:value `` pairs.
- map<string, string> labels = 4 [(yandex.cloud.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-}
-
-message CreateServiceAccountMetadata {
- // ID of the service account that is being created.
- string service_account_id = 1;
-}
-
-message UpdateServiceAccountRequest {
- // ID of the ServiceAccount resource to update.
- // To get the service account ID, use a [ServiceAccountService.List] request.
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-
- // Field mask that specifies which fields of the ServiceAccount resource are going to be updated.
- google.protobuf.FieldMask update_mask = 2;
-
- // Name of the service account.
- // The name must be unique within the cloud.
- string name = 3 [(required) = true, (pattern) = "|[a-z][-a-z0-9]{1,61}[a-z0-9]"];
-
- // Description of the service account.
- string description = 4 [(length) = "<=256"];
-
- // Resource labels as `` key:value `` pairs.
- map<string, string> labels = 5 [(yandex.cloud.size) = "<=64", (length) = "<=63", (pattern) = "[-_0-9a-z]*", (map_key).length = "1-63", (map_key).pattern = "[a-z][-_0-9a-z]*"];
-}
-
-message UpdateServiceAccountMetadata {
- // ID of the ServiceAccount resource that is being updated.
- string service_account_id = 1;
-}
-
-message DeleteServiceAccountRequest {
- // ID of the service account to delete.
- // To get the service account ID, use a [ServiceAccountService.List] request.
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-}
-
-message DeleteServiceAccountMetadata {
- // ID of the service account that is being deleted.
- string service_account_id = 1;
-}
-
-message ListServiceAccountOperationsRequest {
- // ID of the ServiceAccount resource to list operations for.
- string service_account_id = 1 [(required) = true, (length) = "<=50"];
-
- // The maximum number of results per page to return. If the number of available
- // results is larger than [page_size], the service returns a [ListServiceAccountOperationsResponse.next_page_token]
- // that can be used to get the next page of results in subsequent list requests.
- // Default value: 100.
- int64 page_size = 2 [(value) = "<=1000"];
-
- // Page token. To get the next page of results, set [page_token]
- // to the [ListServiceAccountOperationsResponse.next_page_token]
- // returned by a previous list request.
- string page_token = 3 [(length) = "<=100"];
-}
-
-message ListServiceAccountOperationsResponse {
- // List of operations for the specified service account.
- repeated operation.Operation operations = 1;
-
- // This token allows you to get the next page of results for list requests. If the number of results
- // is larger than [ListServiceAccountOperationsRequest.page_size], use the [next_page_token] as the value
- // for the [ListServiceAccountOperationsRequest.page_token] query parameter in the next list request.
- // Each subsequent list request will have its own [next_page_token] to continue paging through the results.
- string next_page_token = 2;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account.proto
deleted file mode 100644
index 7fe7fda234..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account.proto
+++ /dev/null
@@ -1,47 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "yandex/cloud/validation.proto";
-
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// Currently represents only [Yandex account](/docs/iam/concepts/#passport).
-message UserAccount {
- // ID of the user account.
- string id = 1;
-
- oneof user_account {
- option (exactly_one) = true;
- // A YandexPassportUserAccount resource.
- YandexPassportUserAccount yandex_passport_user_account = 2;
- // A SAML federated user.
- SamlUserAccount saml_user_account = 3;
- }
-}
-
-// A YandexPassportUserAccount resource.
-// For more information, see [Yandex account](/docs/iam/concepts/#passport).
-message YandexPassportUserAccount {
- // Login of the Yandex user account.
- string login = 1;
- // Default email of the Yandex user account.
- string default_email = 2;
-}
-
-// A SAML federated user.
-// For more information, see [federations](/docs/iam/concepts/users/saml-federations).
-message SamlUserAccount {
- message Attribute {
- repeated string value = 1;
- }
- // ID of the federation that the federation belongs to.
- string federation_id = 1 [(required) = true, (length) = "<=50"];
- // Name Id of the SAML federated user.
- // The name is unique within the federation. 1-256 characters long.
- string name_id = 2 [(required) = true, (length) = "1-256"];
- // Additional attributes of the SAML federated user.
- map <string, Attribute> attributes = 3;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account_service.proto
deleted file mode 100644
index 102f664dea..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/user_account_service.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/iam/v1/user_account.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing user accounts. Currently applicable only for [Yandex accounts](/docs/iam/concepts/#passport).
-service UserAccountService {
- // Returns the specified UserAccount resource.
- rpc Get (GetUserAccountRequest) returns (UserAccount) {
- option (google.api.http) = { get: "/iam/v1/userAccounts/{user_account_id}" };
- }
-}
-
-message GetUserAccountRequest {
- // ID of the UserAccount resource to return.
- string user_account_id = 1 [(required) = true, (length) = "<=50"];
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/yandex_passport_user_account_service.proto b/cloud/bitbucket/public-api/yandex/cloud/iam/v1/yandex_passport_user_account_service.proto
deleted file mode 100644
index eabd7ae463..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/iam/v1/yandex_passport_user_account_service.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.iam.v1;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/iam/v1/user_account.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/iam/v1;iam";
-option java_package = "yandex.cloud.api.iam.v1";
-
-// A set of methods for managing YandexPassportUserAccount resources.
-service YandexPassportUserAccountService {
- // Returns the specified YandexPassportUserAccount resource.
- rpc GetByLogin (GetUserAccountByLoginRequest) returns (UserAccount) {
- option (google.api.http) = { get: "/iam/v1/yandexPassportUserAccounts:byLogin" };
- }
-}
-
-message GetUserAccountByLoginRequest {
- // Login of the YandexPassportUserAccount resource to return.
- string login = 1 [(required) = true];
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/imports.proto b/cloud/bitbucket/public-api/yandex/cloud/imports.proto
deleted file mode 100644
index a6774f180b..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/imports.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud;
-
-// This file contains all imports that may not be required by API itself, but
-// should present in PB-descriptor for grpc-json conversion. The most vivid
-// examples are messages defined in google/rpc/error_details.proto, which
-// may appear in status details but are not mentioned anywhere else in API as
-// import.
-
-import "google/rpc/status.proto";
-import "google/rpc/error_details.proto";
-import "google/rpc/code.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud;cloud";
-option java_package = "yandex.cloud.api";
-
-message Dummy {
- // Reference all includes to avoid 'unused' warnings.
- google.rpc.Status status = 1;
- google.rpc.Code code = 2;
- google.rpc.RequestInfo request_info = 3;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/operation/CMakeLists.txt b/cloud/bitbucket/public-api/yandex/cloud/operation/CMakeLists.txt
deleted file mode 100644
index f7629ae82c..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/operation/CMakeLists.txt
+++ /dev/null
@@ -1,56 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yandex-cloud-operation)
-set_property(TARGET yandex-cloud-operation PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET yandex-cloud-operation PROPERTY
- PROTO_NAMESPACE cloud/bitbucket/public-api
-)
-target_include_directories(yandex-cloud-operation PUBLIC
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_include_directories(yandex-cloud-operation PRIVATE
- ${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_link_libraries(yandex-cloud-operation PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-googleapis-common-protos
- cloud-api-tools
- public-api-yandex-cloud
- contrib-libs-protobuf
-)
-target_proto_messages(yandex-cloud-operation PRIVATE
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/operation/operation.proto
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api/yandex/cloud/operation/operation_service.proto
-)
-target_proto_addincls(yandex-cloud-operation
- ./cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/common-api
- ${CMAKE_SOURCE_DIR}/cloud/bitbucket/public-api
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(yandex-cloud-operation
- --cpp_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/cloud/bitbucket/public-api
-)
-target_proto_plugin(yandex-cloud-operation
- grpc_cpp
- grpc_cpp
-)
diff --git a/cloud/bitbucket/public-api/yandex/cloud/operation/openapi-meta.yaml b/cloud/bitbucket/public-api/yandex/cloud/operation/openapi-meta.yaml
deleted file mode 100644
index 44aa30cf21..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/operation/openapi-meta.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-info:
- version:
- title: Yandex.Cloud Operations API
- description: This API reference is organized by resource. Actions are performed by sending HTTP requests to resource URLs or making RPC calls. For more information about Yandex.Cloud API architecture, see [API Concepts](/docs/api-design-guide/).
- x-y-docUri: api-ref
- x-y-baseUrl: "https://operation.api.cloud.yandex.net"
-docName: "/docs/operation/api-ref"
diff --git a/cloud/bitbucket/public-api/yandex/cloud/operation/operation.proto b/cloud/bitbucket/public-api/yandex/cloud/operation/operation.proto
deleted file mode 100644
index 053029f330..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/operation/operation.proto
+++ /dev/null
@@ -1,55 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.operation;
-
-import "google/protobuf/any.proto";
-import "google/rpc/status.proto";
-import "google/protobuf/timestamp.proto";
-
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/operation;operation";
-option java_package = "yandex.cloud.api.operation";
-
-// An Operation resource. For more information, see [Operation](/docs/api-design-guide/concepts/operation).
-message Operation {
- // ID of the operation.
- string id = 1;
-
- // Description of the operation. 0-256 characters long.
- string description = 2; // ex: Create VM, Stop VM, Delete Disk, Snapshot Disk, etc
-
- // Creation timestamp.
- google.protobuf.Timestamp created_at = 3;
-
- // ID of the user or service account who initiated the operation.
- string created_by = 4;
-
- // The time when the Operation resource was last modified.
- google.protobuf.Timestamp modified_at = 5;
-
- // If the value is `false`, it means the operation is still in progress.
- // If `true`, the operation is completed, and either `error` or `response` is available.
- bool done = 6;
-
- // Service-specific metadata associated with the operation.
- // It typically contains the ID of the target resource that the operation is performed on.
- // Any method that returns a long-running operation should document the metadata type, if any.
- google.protobuf.Any metadata = 7;
-
- // The operation result.
- // If `done == false` and there was no failure detected, neither `error` nor `response` is set.
- // If `done == false` and there was a failure detected, `error` is set.
- // If `done == true`, exactly one of `error` or `response` is set.
- oneof result {
- // The error result of the operation in case of failure or cancellation.
- google.rpc.Status error = 8;
-
- // The normal response of the operation in case of success.
- // If the original method returns no data on success, such as Delete,
- // the response is [google.protobuf.Empty].
- // If the original method is the standard Create/Update,
- // the response should be the target resource of the operation.
- // Any method that returns a long-running operation should document the response type, if any.
- google.protobuf.Any response = 9;
- }
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/operation/operation_service.proto b/cloud/bitbucket/public-api/yandex/cloud/operation/operation_service.proto
deleted file mode 100644
index cdda823e36..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/operation/operation_service.proto
+++ /dev/null
@@ -1,37 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud.operation;
-
-import "google/api/annotations.proto";
-import "yandex/cloud/api/tools/options.proto";
-import "yandex/cloud/operation/operation.proto";
-import "yandex/cloud/validation.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud/operation;operation";
-option java_package = "yandex.cloud.api.operation";
-
-// A set of methods for managing operations for asynchronous API requests.
-service OperationService {
- // Returns the specified Operation resource.
- rpc Get (GetOperationRequest) returns (Operation) {
- option (google.api.http) = { get: "/operations/{operation_id}" };
- }
-
- // Cancels the specified operation.
- //
- // Note that currently Yandex Object Storage API does not support cancelling operations.
- rpc Cancel (CancelOperationRequest) returns (Operation) {
- option (google.api.http) = { get: "/operations/{operation_id}:cancel" };
- option (yandex.cloud.api.tools.method).lint_skip.http_verb = true;
- }
-}
-
-message GetOperationRequest {
- // ID of the Operation resource to return.
- string operation_id = 1 [(required) = true];
-}
-
-message CancelOperationRequest {
- // ID of the operation to cancel.
- string operation_id = 1 [(required) = true];
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/validation.proto b/cloud/bitbucket/public-api/yandex/cloud/validation.proto
deleted file mode 100644
index e3f5ffca74..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/validation.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-syntax = "proto3";
-
-package yandex.cloud;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "a.yandex-team.ru/cloud/bitbucket/public-api/yandex/cloud;cloud";
-option java_package = "yandex.cloud.api";
-
-extend google.protobuf.OneofOptions {
- bool exactly_one = 101400;
-}
-
-extend google.protobuf.FieldOptions {
- bool required = 101501;
- string pattern = 101502;
- string value = 101503;
- string size = 101504;
- string length = 101505;
- bool unique = 101506;
- MapKeySpec map_key = 101510;
- string bytes = 101511;
-}
-
-message MapKeySpec {
- string value = 1;
- string pattern = 2;
- string length = 3;
-}
diff --git a/cloud/bitbucket/public-api/yandex/cloud/validation.yadoc.yaml b/cloud/bitbucket/public-api/yandex/cloud/validation.yadoc.yaml
deleted file mode 100644
index ab37b83de4..0000000000
--- a/cloud/bitbucket/public-api/yandex/cloud/validation.yadoc.yaml
+++ /dev/null
@@ -1,127 +0,0 @@
-# NOTE(skipor): text will be placed in description, which is CommonMark Markdown
-# so escape regular expressions using `` <pattern> `` escaping, to escape even '`' characters.
-
-required: Required.
-
-pattern:
- usual: Value must match the regular expression `` $PATTERN ``.
- repeated: Each value must match the regular expression `` $PATTERN ``.
- map_value: Each value must match the regular expression `` $PATTERN ``.
-
-value:
- usual:
- "<=": The maximum value is $MAX_VALUE.
- "<": Value must be less than $MAX_VALUE.
- ">=": The minimum value is $MIN_VALUE.
- ">": Value must be greater than $MIN_VALUE.
- "min-max": Acceptable values are $MIN_VALUE to $MAX_VALUE, inclusive.
- "=": Value must be equal to $VALUE.
- "list": Value must be one of $FIRST_VALUES or $LAST_VALUE.
- repeated:
- "<=": Each value must be less than or equal to $MAX_VALUE.
- "<": Each value must be less than $MAX_VALUE.
- ">=": Each value must be greater than or equal to $MIN_VALUE.
- ">": Each value must be greater than $MIN_VALUE.
- "min-max": Acceptable values are $MIN_VALUE to $MAX_VALUE, inclusive.
- "=": Each value must be equal to $VALUE.
- "list": Each value must be one of $FIRST_VALUES or $LAST_VALUE.
- map_value:
- "<=": Each value must be less than or equal to $MAX_VALUE.
- "<": Each value must be less than $MAX_VALUE.
- ">=": Each value must be greater than or equal to $MIN_VALUE.
- ">": Each value must be greater than $MIN_VALUE.
- "min-max": Acceptable values are $MIN_VALUE to $MAX_VALUE, inclusive.
- "=": Each value must be equal to $VALUE.
- "list": Each value must be one of $FIRST_VALUES or $LAST_VALUE.
-
-size:
- repeated:
- "<=": The maximum number of elements is $MAX_VALUE.
- "<": The number of elements must be less than $MAX_VALUE.
- ">=": The minimum number of elements is $MIN_VALUE.
- ">0": Must contain at least one element.
- ">": The number of elements must be greater than $MIN_VALUE.
- "min-max": The number of elements must be in the range $MIN_VALUE-$MAX_VALUE.
- "1": Must contain exactly 1 element.
- "=": The number of elements must be exactly $VALUE.
- "list": The number of elements must be one of $FIRST_VALUES or $LAST_VALUE.
- map_value:
- "<=": No more than $MAX_VALUE per resource.
- "<": Less than $MAX_VALUE per resource.
- ">=": No less than $MIN_VALUE per resource.
- ">0": At least one per resource.
- ">": More than $MIN_VALUE per resource.
- "min-max": $MIN_VALUE-$MAX_VALUE per resource.
- "=": Exactly $VALUE per resource.
- "list": $FIRST_VALUES or $LAST_VALUE per resource.
-
-length:
- usual:
- "<=": The maximum string length in characters is $MAX_VALUE.
- "<": The string length in characters must be less than $MAX_VALUE.
- ">=": The minimum string length in characters is $MIN_VALUE.
- ">": The string length in characters must be greater than $MIN_VALUE.
- "min-max": The string length in characters must be $MIN_VALUE-$MAX_VALUE.
- "=": The string length in characters must be equal to $VALUE.
- "list": The string length in characters must be equal to one of $FIRST_VALUES or $LAST_VALUE.
- repeated:
- "<=": The maximum string length in characters for each value is $MAX_VALUE.
- "<": The string length in characters for each value must be less than $MAX_VALUE.
- ">=": The minimum string length in characters for each value is $MIN_VALUE.
- ">": The string length in characters for each value must be greater than $MIN_VALUE.
- "min-max": The string length in characters for each value must be $MIN_VALUE-$MAX_VALUE.
- "=": The string length in characters for each value must be equal to $VALUE.
- "list": The string length in characters for each value must be equal to one of $FIRST_VALUES or $LAST_VALUE.
- map_value:
- "<=": The maximum string length in characters for each value is $MAX_VALUE.
- "<": The string length in characters for each value must be less than $MAX_VALUE.
- ">=": The minimum string length in characters for each value is $MIN_VALUE.
- ">": The string length in characters for each value must be greater than $MIN_VALUE.
- "min-max": The string length in characters for each value must be $MIN_VALUE-$MAX_VALUE.
- "=": The string length in characters for each value must be equal to $VALUE.
- "list": The string length in characters for each value must be equal to one of $FIRST_VALUES or $LAST_VALUE.
-
-bytes:
- usual:
- "<=": The maximum size in bytes is $MAX_VALUE.
- "<": The size in bytes must be less than $MAX_VALUE.
- ">=": The minimum size in bytes is $MIN_VALUE.
- ">": The size in bytes must be greater than $MIN_VALUE.
- "min-max": The size in bytes must be $MIN_VALUE-$MAX_VALUE.
- "=": The size in bytes must be equal to $VALUE.
- "list": The size in bytes must be equal to one of $FIRST_VALUES or $LAST_VALUE.
- repeated:
- "<=": The maximum size in bytes for each value is $MAX_VALUE.
- "<": The size in bytes for each value must be less than $MAX_VALUE.
- ">=": The minimum size in bytes for each value is $MIN_VALUE.
- ">": The size in bytes for each value must be greater than $MIN_VALUE.
- "min-max": The size in bytes for each value must be $MIN_VALUE-$MAX_VALUE.
- "=": The size in bytes for each value must be equal to $VALUE.
- "list": The size in bytes for each value must be equal to one of $FIRST_VALUES or $LAST_VALUE.
- map_value:
- "<=": The maximum size in bytes for each value is $MAX_VALUE.
- "<": The size in bytes for each value must be less than $MAX_VALUE.
- ">=": The minimum size in bytes for each value is $MIN_VALUE.
- ">": The size in bytes for each value must be greater than $MIN_VALUE.
- "min-max": The size in bytes for each value must be $MIN_VALUE-$MAX_VALUE.
- "=": The size in bytes for each value must be equal to $VALUE.
- "list": The size in bytes for each value must be equal to one of $FIRST_VALUES or $LAST_VALUE.
-
-map_key:
- value:
- "<=": Each key must be less than or equal to $MAX_VALUE.
- "<": Each key must be less than $MAX_VALUE.
- ">=": Each key must be greater than or equal to $MIN_VALUE.
- ">": Each key must be greater than $MIN_VALUE.
- "min-max": Acceptable keys are $MIN_VALUE to $MAX_VALUE, inclusive.
- "=": Each key must be equal to $VALUE.
- "list": Each key must be one of $FIRST_VALUES or $LAST_VALUE.
- pattern: Each key must match the regular expression `` $PATTERN ``.
- length:
- "<=": The maximum string length in characters for each key is $MAX_VALUE.
- "<": The string length in characters for each key must be less than $MAX_VALUE.
- ">=": The minimum string length in characters for each key is $MIN_VALUE.
- ">": The string length in characters for each key must be greater than $MIN_VALUE.
- "min-max": The string length in characters for each key must be $MIN_VALUE-$MAX_VALUE.
- "=": The string length in characters for each key must be equal to $VALUE.
- "list": The string length in characters for each key must be equal to one of $FIRST_VALUES or $LAST_VALUE.
diff --git a/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.copyrights.report b/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.copyrights.report
deleted file mode 100644
index 8b7f8626d2..0000000000
--- a/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.copyrights.report
+++ /dev/null
@@ -1,192 +0,0 @@
-# File format ($ symbol means the beginning of a line):
-#
-# $ # this message
-# $ # =======================
-# $ # comments (all commentaries should starts with some number of spaces and # symbol)
-# ${action} {license id} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make
-# ${all_file_action} filename
-# $ # user commentaries (many lines)
-# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/3/ya.make
-# ${all_file_action} filename
-# $ # user commentaries
-# $ generated description
-# $ ...
-#
-# You can modify action, all_file_action and add commentaries
-# Available actions:
-# keep - keep license in contrib and use in credits
-# skip - skip license
-# remove - remove all files with this license
-# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file
-#
-# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory)
-# We suppose that that files can contain some license info
-# Available all file actions:
-# FILE_IGNORE - ignore file (do nothing)
-# FILE_INCLUDE - include all file data into licenses text file
-# =======================
-
-KEEP COPYRIGHT_SERVICE_LABEL 05bdd09fb9fdb384a61f2eb54df462d6
-BELONGS ya.make
- License text:
- // Copyright 2016 Google Inc.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/consumer.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 2531f49b086b54e88aae9438d8a9d63f
-BELONGS ya.make
- License text:
- // Copyright 2020 Google LLC
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/client.proto [1:1]
- google/api/field_behavior.proto [1:1]
- google/api/resource.proto [1:1]
- google/rpc/code.proto [1:1]
- google/rpc/context/attribute_context.proto [1:1]
- google/rpc/error_details.proto [1:1]
- google/rpc/status.proto [1:1]
- google/type/latlng.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 30222adbd0b829894165c0b67a2f8f49
-BELONGS ya.make
- License text:
- // Copyright (c) 2015, Google Inc.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/annotations.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 31fab26359e7ce5661ae59c43e7d60cb
-BELONGS ya.make
- License text:
- // Copyright 2018 Google LLC
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/http.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 498cf2f7854ebbf6af390ac0fc190ee7
-BELONGS ya.make
- License text:
- // Copyright 2021 Google LLC
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/routing.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 6391d45c5ab2c1a5ff8c29e13d3bdfdf
-BELONGS ya.make
- License text:
- // Copyright 2018 Google LLC.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/auth.proto [1:1]
- google/api/backend.proto [1:1]
- google/api/billing.proto [1:1]
- google/api/config_change.proto [1:1]
- google/api/control.proto [1:1]
- google/api/documentation.proto [1:1]
- google/api/endpoint.proto [1:1]
- google/api/httpbody.proto [1:1]
- google/api/label.proto [1:1]
- google/api/log.proto [1:1]
- google/api/logging.proto [1:1]
- google/api/metric.proto [1:1]
- google/api/monitored_resource.proto [1:1]
- google/api/monitoring.proto [1:1]
- google/api/source_info.proto [1:1]
- google/api/system_parameter.proto [1:1]
- google/api/usage.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 6499e2ad737f62db5558c81fbd2749a7
-BELONGS ya.make
- License text:
- // Copyright 2017 Google Inc.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/context.proto [1:1]
- google/api/quota.proto [1:1]
- google/api/service.proto [1:1]
- google/logging/type/http_request.proto [1:1]
- google/logging/type/log_severity.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL 79b98accf2a4f2be03be86ac314d9535
-BELONGS ya.make
- License text:
- // Copyright 2019 Google LLC.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/api/distribution.proto [1:1]
- google/api/launch_stage.proto [1:1]
- google/longrunning/operations.proto [1:1]
- google/type/calendar_period.proto [1:1]
- google/type/color.proto [1:1]
- google/type/date.proto [1:1]
- google/type/datetime.proto [1:1]
- google/type/dayofweek.proto [1:1]
- google/type/expr.proto [1:1]
- google/type/fraction.proto [1:1]
- google/type/money.proto [1:1]
- google/type/month.proto [1:1]
- google/type/postal_address.proto [1:1]
- google/type/quaternion.proto [1:1]
- google/type/timeofday.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL c06f0823493d2ec03ec723004d287aab
-BELONGS ya.make
- License text:
- // Copyright 2020 Google LLC.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/gapic/metadata/gapic_metadata.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL de62d4ef47d15bdcbe773ca452bc3c3a
-BELONGS ya.make
- License text:
- // Copyright 2021 Google LLC.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/cloud/extended_operations.proto [1:1]
-
-KEEP COPYRIGHT_SERVICE_LABEL e618dd164932f955bb8121f5f486e841
-BELONGS ya.make
- License text:
- // Copyright 2022 Google LLC
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- google/cloud/location/locations.proto [1:1]
diff --git a/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.licenses.report b/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.licenses.report
deleted file mode 100644
index 6df2244bcc..0000000000
--- a/contrib/libs/googleapis-common-protos/.yandex_meta/devtools.licenses.report
+++ /dev/null
@@ -1,128 +0,0 @@
-# File format ($ symbol means the beginning of a line):
-#
-# $ # this message
-# $ # =======================
-# $ # comments (all commentaries should starts with some number of spaces and # symbol)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make
-# ${all_file_action} filename
-# $ # user commentaries (many lines)
-# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/3/ya.make
-# ${all_file_action} filename
-# $ # user commentaries
-# $ generated description
-# $ ...
-#
-# You can modify action, all_file_action and add commentaries
-# Available actions:
-# keep - keep license in contrib and use in credits
-# skip - skip license
-# remove - remove all files with this license
-# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file
-#
-# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory)
-# We suppose that that files can contain some license info
-# Available all file actions:
-# FILE_IGNORE - ignore file (do nothing)
-# FILE_INCLUDE - include all file data into licenses text file
-# =======================
-
-KEEP Apache-2.0 2b42edef8fa55315f34f2370b4715ca9
-BELONGS ya.make
- Note: matched license text is too long. Read it in the source files.
- Scancode info:
- Original SPDX id: Apache-2.0
- Score : 100.00
- Match type : TEXT
- Links : http://www.apache.org/licenses/, http://www.apache.org/licenses/LICENSE-2.0, https://spdx.org/licenses/Apache-2.0
- Files with this license:
- LICENSE [2:202]
-
-KEEP Apache-2.0 4248d6c827c367ff9f1b5e59ca41408e
-BELONGS ya.make
- Note: matched license text is too long. Read it in the source files.
- Scancode info:
- Original SPDX id: Apache-2.0
- Score : 100.00
- Match type : NOTICE
- Links : http://www.apache.org/licenses/, http://www.apache.org/licenses/LICENSE-2.0, https://spdx.org/licenses/Apache-2.0
- Files with this license:
- google/api/annotations.proto [3:13]
- google/api/auth.proto [3:13]
- google/api/backend.proto [3:13]
- google/api/billing.proto [3:13]
- google/api/client.proto [3:13]
- google/api/config_change.proto [3:13]
- google/api/consumer.proto [3:13]
- google/api/context.proto [3:13]
- google/api/control.proto [3:13]
- google/api/distribution.proto [3:13]
- google/api/documentation.proto [3:13]
- google/api/endpoint.proto [3:13]
- google/api/field_behavior.proto [3:13]
- google/api/http.proto [3:13]
- google/api/httpbody.proto [3:13]
- google/api/label.proto [3:13]
- google/api/launch_stage.proto [3:13]
- google/api/log.proto [3:13]
- google/api/logging.proto [3:13]
- google/api/metric.proto [3:13]
- google/api/monitored_resource.proto [3:13]
- google/api/monitoring.proto [3:13]
- google/api/quota.proto [3:13]
- google/api/resource.proto [3:13]
- google/api/routing.proto [3:13]
- google/api/service.proto [3:13]
- google/api/source_info.proto [3:13]
- google/api/system_parameter.proto [3:13]
- google/api/usage.proto [3:13]
- google/cloud/extended_operations.proto [3:13]
- google/cloud/location/locations.proto [3:13]
- google/gapic/metadata/gapic_metadata.proto [3:13]
- google/logging/type/http_request.proto [3:13]
- google/logging/type/log_severity.proto [3:13]
- google/longrunning/operations.proto [3:13]
- google/rpc/code.proto [3:13]
- google/rpc/context/attribute_context.proto [3:13]
- google/rpc/error_details.proto [3:13]
- google/rpc/status.proto [3:13]
- google/type/calendar_period.proto [3:13]
- google/type/color.proto [3:13]
- google/type/date.proto [3:13]
- google/type/datetime.proto [3:13]
- google/type/dayofweek.proto [3:13]
- google/type/expr.proto [3:13]
- google/type/fraction.proto [3:13]
- google/type/latlng.proto [3:13]
- google/type/money.proto [3:13]
- google/type/month.proto [3:13]
- google/type/postal_address.proto [3:13]
- google/type/quaternion.proto [3:13]
- google/type/timeofday.proto [3:13]
-
-SKIP LicenseRef-scancode-generic-cla 5d780ffa423067f23c6a123ae33e7c18
-BELONGS ya.make
- License text:
- \## Contributor License Agreement
- Scancode info:
- Original SPDX id: LicenseRef-scancode-generic-cla
- Score : 16.00
- Match type : NOTICE
- Links : https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/generic-cla.LICENSE
- Files with this license:
- CONTRIBUTING.md [6:6]
-
-SKIP LicenseRef-scancode-generic-cla d629c1791f1024cda2ec24fa16773bcd
-BELONGS ya.make
- License text:
- Contributions to this project must be accompanied by a Contributor License
- Agreement (CLA). You (or your employer) retain the copyright to your
- Scancode info:
- Original SPDX id: LicenseRef-scancode-generic-cla
- Score : 22.00
- Match type : NOTICE
- Links : https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/generic-cla.LICENSE
- Files with this license:
- CONTRIBUTING.md [8:9]
diff --git a/contrib/libs/googleapis-common-protos/.yandex_meta/licenses.list.txt b/contrib/libs/googleapis-common-protos/.yandex_meta/licenses.list.txt
deleted file mode 100644
index cdd8bd22f7..0000000000
--- a/contrib/libs/googleapis-common-protos/.yandex_meta/licenses.list.txt
+++ /dev/null
@@ -1,259 +0,0 @@
-====================Apache-2.0====================
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-====================Apache-2.0====================
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
-====================COPYRIGHT====================
-// Copyright (c) 2015, Google Inc.
-
-
-====================COPYRIGHT====================
-// Copyright 2016 Google Inc.
-
-
-====================COPYRIGHT====================
-// Copyright 2017 Google Inc.
-
-
-====================COPYRIGHT====================
-// Copyright 2018 Google LLC
-
-
-====================COPYRIGHT====================
-// Copyright 2018 Google LLC.
-
-
-====================COPYRIGHT====================
-// Copyright 2019 Google LLC.
-
-
-====================COPYRIGHT====================
-// Copyright 2020 Google LLC
-
-
-====================COPYRIGHT====================
-// Copyright 2020 Google LLC.
-
-
-====================COPYRIGHT====================
-// Copyright 2021 Google LLC
-
-
-====================COPYRIGHT====================
-// Copyright 2021 Google LLC.
-
-
-====================COPYRIGHT====================
-// Copyright 2022 Google LLC
diff --git a/contrib/libs/googleapis-common-protos/CHANGELOG.md b/contrib/libs/googleapis-common-protos/CHANGELOG.md
deleted file mode 100644
index 6d6264cbe8..0000000000
--- a/contrib/libs/googleapis-common-protos/CHANGELOG.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Changelog
-
-## [1.55.0](https://github.com/googleapis/python-api-common-protos/compare/v1.54.0...v1.55.0) (2022-02-15)
-
-
-### Features
-
-* add location proto files. ([#84](https://github.com/googleapis/python-api-common-protos/issues/84)) ([9a33e56](https://github.com/googleapis/python-api-common-protos/commit/9a33e56ac6a07a2e717edc55a39fa7cf2f9eec15))
-
-## [1.54.0](https://www.github.com/googleapis/python-api-common-protos/compare/v1.53.0...v1.54.0) (2021-12-07)
-
-
-### Features
-
-* add extended_operations.proto ([#77](https://www.github.com/googleapis/python-api-common-protos/issues/77)) ([bc85849](https://www.github.com/googleapis/python-api-common-protos/commit/bc85849e21494b267d87cd6dc5d0a0e23e012470))
-* add google/api/routing.proto ([#75](https://www.github.com/googleapis/python-api-common-protos/issues/75)) ([1ae0bbc](https://www.github.com/googleapis/python-api-common-protos/commit/1ae0bbcc9747af4dd467e7a246c1a2a4cd5ef2ec))
-
-## [1.53.0](https://www.github.com/googleapis/python-api-common-protos/compare/v1.52.0...v1.53.0) (2021-02-25)
-
-
-### Features
-
-* add `google.api.ResourceDescriptor.Style` ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-* add API method signatures to longrunning operations ([8de7ae2](https://www.github.com/googleapis/python-api-common-protos/commit/8de7ae28dfe5dd4d0cb99dd3b89a8f1e614bbe6d))
-* add gapic_metadata_pb2 ([#38](https://www.github.com/googleapis/python-api-common-protos/issues/38)) ([8de7ae2](https://www.github.com/googleapis/python-api-common-protos/commit/8de7ae28dfe5dd4d0cb99dd3b89a8f1e614bbe6d))
-* add UNORDERED_LIST to field options ([8de7ae2](https://www.github.com/googleapis/python-api-common-protos/commit/8de7ae28dfe5dd4d0cb99dd3b89a8f1e614bbe6d))
-* add WaitOperation method to longrunning operations ([8de7ae2](https://www.github.com/googleapis/python-api-common-protos/commit/8de7ae28dfe5dd4d0cb99dd3b89a8f1e614bbe6d))
-* require python >=3.6 and ([#31](https://www.github.com/googleapis/python-api-common-protos/issues/31)) ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-
-
-### Bug Fixes
-
-* add `create_key` to FieldDescriptors ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-* Generate gRPC files for long-running operations ([#13](https://www.github.com/googleapis/python-api-common-protos/issues/13)) ([a9ce288](https://www.github.com/googleapis/python-api-common-protos/commit/a9ce28840ddfec712da5b296f43e6c3131840db4))
-
-
-### Documentation
-
-* add link to PyPI ([#10](https://www.github.com/googleapis/python-api-common-protos/issues/10)) ([3f79402](https://www.github.com/googleapis/python-api-common-protos/commit/3f7940226b0e22aef31b82c8dc2196aa25b48a3f))
-
-## [1.53.0dev1](https://www.github.com/googleapis/python-api-common-protos/compare/v1.52.0...v1.53.0dev1) (2021-01-27)
-
-
-### Features
-
-* add `google.api.ResourceDescriptor.Style` ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-* require python >=3.6 and ([#31](https://www.github.com/googleapis/python-api-common-protos/issues/31)) ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-
-
-### Bug Fixes
-
-* add `create_key` to FieldDescriptors ([4ce679c](https://www.github.com/googleapis/python-api-common-protos/commit/4ce679cd49771946bf781108e92e07cdf04a61eb))
-* Generate gRPC files for long-running operations ([#13](https://www.github.com/googleapis/python-api-common-protos/issues/13)) ([a9ce288](https://www.github.com/googleapis/python-api-common-protos/commit/a9ce28840ddfec712da5b296f43e6c3131840db4))
-
-
-### Documentation
-
-* add link to PyPI ([#10](https://www.github.com/googleapis/python-api-common-protos/issues/10)) ([3f79402](https://www.github.com/googleapis/python-api-common-protos/commit/3f7940226b0e22aef31b82c8dc2196aa25b48a3f))
-
-## 1.52.0 (2020-06-04)
-
-
-### Features
-
-* create api-common-protos repo for python common protos ([4ef4b0d](https://www.github.com/googleapis/python-api-common-protos/commit/4ef4b0d177136bfbd19f4c00ccf2f6d7eaccb153))
diff --git a/contrib/libs/googleapis-common-protos/CMakeLists.txt b/contrib/libs/googleapis-common-protos/CMakeLists.txt
deleted file mode 100644
index 5ceb3387d9..0000000000
--- a/contrib/libs/googleapis-common-protos/CMakeLists.txt
+++ /dev/null
@@ -1,100 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(contrib-libs-googleapis-common-protos)
-set_property(TARGET contrib-libs-googleapis-common-protos PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-set_property(TARGET contrib-libs-googleapis-common-protos PROPERTY
- PROTO_NAMESPACE contrib/libs/googleapis-common-protos
-)
-target_include_directories(contrib-libs-googleapis-common-protos PUBLIC
- ${CMAKE_BINARY_DIR}/contrib/libs/googleapis-common-protos
-)
-target_include_directories(contrib-libs-googleapis-common-protos PRIVATE
- ${CMAKE_BINARY_DIR}/contrib/libs/googleapis-common-protos
-)
-target_link_libraries(contrib-libs-googleapis-common-protos PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-protobuf
-)
-target_proto_messages(contrib-libs-googleapis-common-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/annotations.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/auth.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/backend.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/billing.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/client.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/config_change.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/consumer.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/context.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/control.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/distribution.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/documentation.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/endpoint.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/field_behavior.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/http.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/httpbody.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/label.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/launch_stage.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/log.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/logging.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/metric.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/monitored_resource.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/monitoring.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/quota.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/resource.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/routing.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/service.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/source_info.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/system_parameter.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/api/usage.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/cloud/extended_operations.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/cloud/location/locations.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/gapic/metadata/gapic_metadata.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/logging/type/http_request.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/logging/type/log_severity.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/longrunning/operations.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/rpc/code.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/rpc/context/attribute_context.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/rpc/error_details.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/rpc/status.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/calendar_period.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/color.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/date.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/datetime.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/dayofweek.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/expr.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/fraction.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/latlng.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/money.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/month.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/postal_address.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/quaternion.proto
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos/google/type/timeofday.proto
-)
-target_proto_addincls(contrib-libs-googleapis-common-protos
- ./contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/googleapis-common-protos
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(contrib-libs-googleapis-common-protos
- --cpp_out=${CMAKE_BINARY_DIR}/contrib/libs/googleapis-common-protos
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/contrib/libs/googleapis-common-protos
-)
-target_proto_plugin(contrib-libs-googleapis-common-protos
- grpc_cpp
- grpc_cpp
-)
diff --git a/contrib/libs/googleapis-common-protos/CODE_OF_CONDUCT.md b/contrib/libs/googleapis-common-protos/CODE_OF_CONDUCT.md
deleted file mode 100644
index 46b2a08ea6..0000000000
--- a/contrib/libs/googleapis-common-protos/CODE_OF_CONDUCT.md
+++ /dev/null
@@ -1,43 +0,0 @@
-# Contributor Code of Conduct
-
-As contributors and maintainers of this project,
-and in the interest of fostering an open and welcoming community,
-we pledge to respect all people who contribute through reporting issues,
-posting feature requests, updating documentation,
-submitting pull requests or patches, and other activities.
-
-We are committed to making participation in this project
-a harassment-free experience for everyone,
-regardless of level of experience, gender, gender identity and expression,
-sexual orientation, disability, personal appearance,
-body size, race, ethnicity, age, religion, or nationality.
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery
-* Personal attacks
-* Trolling or insulting/derogatory comments
-* Public or private harassment
-* Publishing other's private information,
-such as physical or electronic
-addresses, without explicit permission
-* Other unethical or unprofessional conduct.
-
-Project maintainers have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct.
-By adopting this Code of Conduct,
-project maintainers commit themselves to fairly and consistently
-applying these principles to every aspect of managing this project.
-Project maintainers who do not follow or enforce the Code of Conduct
-may be permanently removed from the project team.
-
-This code of conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior
-may be reported by opening an issue
-or contacting one or more of the project maintainers.
-
-This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
-available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
diff --git a/contrib/libs/googleapis-common-protos/CONTRIBUTING.md b/contrib/libs/googleapis-common-protos/CONTRIBUTING.md
deleted file mode 100644
index b16a4668cb..0000000000
--- a/contrib/libs/googleapis-common-protos/CONTRIBUTING.md
+++ /dev/null
@@ -1,29 +0,0 @@
-# How to Contribute
-
-We'd love to accept your patches and contributions to this project. There are
-just a few small guidelines you need to follow.
-
-## Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement (CLA). You (or your employer) retain the copyright to your
-contribution; this simply gives us permission to use and redistribute your
-contributions as part of the project. Head over to
-<https://cla.developers.google.com/> to see your current agreements on file or
-to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
-## Code reviews
-
-All submissions, including submissions by project members, require review. We
-use GitHub pull requests for this purpose. Consult
-[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
-information on using pull requests.
-
-## Community Guidelines
-
-This project follows
-[Google's Open Source Community Guidelines](https://opensource.google/conduct/). \ No newline at end of file
diff --git a/contrib/libs/googleapis-common-protos/LICENSE b/contrib/libs/googleapis-common-protos/LICENSE
deleted file mode 100644
index d645695673..0000000000
--- a/contrib/libs/googleapis-common-protos/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/contrib/libs/googleapis-common-protos/README.md b/contrib/libs/googleapis-common-protos/README.md
deleted file mode 100644
index 8fe5fbe677..0000000000
--- a/contrib/libs/googleapis-common-protos/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-
-# Google APIs common protos
-
-[![pypi](https://img.shields.io/pypi/v/googleapis-common-protos.svg)](https://pypi.org/project/googleapis-common-protos/)
-
-
-googleapis-common-protos contains the python classes generated from the common
-protos in the [googleapis/api-common-protos](https://github.com/googleapis/api-common-protos) repository.
diff --git a/contrib/libs/googleapis-common-protos/SECURITY.md b/contrib/libs/googleapis-common-protos/SECURITY.md
deleted file mode 100644
index 8b58ae9c01..0000000000
--- a/contrib/libs/googleapis-common-protos/SECURITY.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Security Policy
-
-To report a security issue, please use [g.co/vulnz](https://g.co/vulnz).
-
-The Google Security Team will respond within 5 working days of your report on g.co/vulnz.
-
-We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.
diff --git a/contrib/libs/googleapis-common-protos/google/api/README.md b/contrib/libs/googleapis-common-protos/google/api/README.md
deleted file mode 100644
index 4aa4b50c90..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-## API Protos
-
-This folder contains the schema of the configuration model for Google's
-internal API serving platform, which handles routing, quotas, monitoring,
-logging, and the like.
-
-Google refers to this configuration colloquially as the "service config",
-and the `service.proto` file in this directory is the entry point for
-understanding these.
-
-## Using these protos
-
-To be honest, we probably open sourced way too much of this (basically by
-accident). There are a couple files in here you are most likely to be
-interested in: `http.proto`, `documentation.proto`, `auth.proto`, and
-`annotations.proto`.
-
-### HTTP and REST
-
-The `http.proto` file contains the `Http` message (which then is wrapped
-in an annotation in `annotations.proto`), which provides a specification
-for REST endpoints and verbs (`GET`, `POST`, etc.) on RPC methods.
-We recommend use of this annotation for describing the relationship
-between RPCs and REST endpoints.
-
-### Documentation
-
-The `documentation.proto` file contains a `Documentation` message which
-provides a mechanism to fully describe an API, allowing a tool to build
-structured documentation artifacts.
-
-### Authentication
-
-The `auth.proto` file contains descriptions of both authentication rules
-and authenticaion providers, allowing you to describe what your services
-expect and accept from clients.
-
-## api-compiler
-
-Google provides a tool called [api-compiler][], which is a tool that accepts
-a protocol buffer descriptor and a YAML file specifying some of the
-options described in `service.proto`, and outputs a much more thorough
-`Service` descriptor based on the information it infers from the protos
-themselves.
-
- [api-compiler]: https://github.com/googleapis/api-compiler
diff --git a/contrib/libs/googleapis-common-protos/google/api/annotations.proto b/contrib/libs/googleapis-common-protos/google/api/annotations.proto
deleted file mode 100644
index 85c361b47f..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/annotations.proto
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (c) 2015, Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/http.proto";
-import "google/protobuf/descriptor.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "AnnotationsProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-extend google.protobuf.MethodOptions {
- // See `HttpRule`.
- HttpRule http = 72295728;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/auth.proto b/contrib/libs/googleapis-common-protos/google/api/auth.proto
deleted file mode 100644
index cdbc1efe90..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/auth.proto
+++ /dev/null
@@ -1,181 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "AuthProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Authentication` defines the authentication configuration for an API.
-//
-// Example for an API targeted for external use:
-//
-// name: calendar.googleapis.com
-// authentication:
-// providers:
-// - id: google_calendar_auth
-// jwks_uri: https://www.googleapis.com/oauth2/v1/certs
-// issuer: https://securetoken.google.com
-// rules:
-// - selector: "*"
-// requirements:
-// provider_id: google_calendar_auth
-message Authentication {
- // A list of authentication rules that apply to individual API methods.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated AuthenticationRule rules = 3;
-
- // Defines a set of authentication providers that a service supports.
- repeated AuthProvider providers = 4;
-}
-
-// Authentication rules for the service.
-//
-// By default, if a method has any authentication requirements, every request
-// must include a valid credential matching one of the requirements.
-// It's an error to include more than one kind of credential in a single
-// request.
-//
-// If a method doesn't have any auth requirements, request credentials will be
-// ignored.
-message AuthenticationRule {
- // Selects the methods to which this rule applies.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // The requirements for OAuth credentials.
- OAuthRequirements oauth = 2;
-
- // If true, the service accepts API keys without any other credential.
- bool allow_without_credential = 5;
-
- // Requirements for additional authentication providers.
- repeated AuthRequirement requirements = 7;
-}
-
-// Configuration for an anthentication provider, including support for
-// [JSON Web Token (JWT)](https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32).
-message AuthProvider {
- // The unique identifier of the auth provider. It will be referred to by
- // `AuthRequirement.provider_id`.
- //
- // Example: "bookstore_auth".
- string id = 1;
-
- // Identifies the principal that issued the JWT. See
- // https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32#section-4.1.1
- // Usually a URL or an email address.
- //
- // Example: https://securetoken.google.com
- // Example: 1234567-compute@developer.gserviceaccount.com
- string issuer = 2;
-
- // URL of the provider's public key set to validate signature of the JWT. See
- // [OpenID Discovery](https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata).
- // Optional if the key set document:
- // - can be retrieved from
- // [OpenID Discovery](https://openid.net/specs/openid-connect-discovery-1_0.html
- // of the issuer.
- // - can be inferred from the email domain of the issuer (e.g. a Google service account).
- //
- // Example: https://www.googleapis.com/oauth2/v1/certs
- string jwks_uri = 3;
-
- // The list of JWT
- // [audiences](https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32#section-4.1.3).
- // that are allowed to access. A JWT containing any of these audiences will
- // be accepted. When this setting is absent, only JWTs with audience
- // "https://[Service_name][google.api.Service.name]/[API_name][google.protobuf.Api.name]"
- // will be accepted. For example, if no audiences are in the setting,
- // LibraryService API will only accept JWTs with the following audience
- // "https://library-example.googleapis.com/google.example.library.v1.LibraryService".
- //
- // Example:
- //
- // audiences: bookstore_android.apps.googleusercontent.com,
- // bookstore_web.apps.googleusercontent.com
- string audiences = 4;
-
- // Redirect URL if JWT token is required but no present or is expired.
- // Implement authorizationUrl of securityDefinitions in OpenAPI spec.
- string authorization_url = 5;
-}
-
-// OAuth scopes are a way to define data and permissions on data. For example,
-// there are scopes defined for "Read-only access to Google Calendar" and
-// "Access to Cloud Platform". Users can consent to a scope for an application,
-// giving it permission to access that data on their behalf.
-//
-// OAuth scope specifications should be fairly coarse grained; a user will need
-// to see and understand the text description of what your scope means.
-//
-// In most cases: use one or at most two OAuth scopes for an entire family of
-// products. If your product has multiple APIs, you should probably be sharing
-// the OAuth scope across all of those APIs.
-//
-// When you need finer grained OAuth consent screens: talk with your product
-// management about how developers will use them in practice.
-//
-// Please note that even though each of the canonical scopes is enough for a
-// request to be accepted and passed to the backend, a request can still fail
-// due to the backend requiring additional scopes or permissions.
-message OAuthRequirements {
- // The list of publicly documented OAuth scopes that are allowed access. An
- // OAuth token containing any of these scopes will be accepted.
- //
- // Example:
- //
- // canonical_scopes: https://www.googleapis.com/auth/calendar,
- // https://www.googleapis.com/auth/calendar.read
- string canonical_scopes = 1;
-}
-
-// User-defined authentication requirements, including support for
-// [JSON Web Token (JWT)](https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32).
-message AuthRequirement {
- // [id][google.api.AuthProvider.id] from authentication provider.
- //
- // Example:
- //
- // provider_id: bookstore_auth
- string provider_id = 1;
-
- // NOTE: This will be deprecated soon, once AuthProvider.audiences is
- // implemented and accepted in all the runtime components.
- //
- // The list of JWT
- // [audiences](https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32#section-4.1.3).
- // that are allowed to access. A JWT containing any of these audiences will
- // be accepted. When this setting is absent, only JWTs with audience
- // "https://[Service_name][google.api.Service.name]/[API_name][google.protobuf.Api.name]"
- // will be accepted. For example, if no audiences are in the setting,
- // LibraryService API will only accept JWTs with the following audience
- // "https://library-example.googleapis.com/google.example.library.v1.LibraryService".
- //
- // Example:
- //
- // audiences: bookstore_android.apps.googleusercontent.com,
- // bookstore_web.apps.googleusercontent.com
- string audiences = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/backend.proto b/contrib/libs/googleapis-common-protos/google/api/backend.proto
deleted file mode 100644
index dd22919d07..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/backend.proto
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "BackendProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Backend` defines the backend configuration for a service.
-message Backend {
- // A list of API backend rules that apply to individual API methods.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated BackendRule rules = 1;
-}
-
-// A backend rule provides configuration for an individual API element.
-message BackendRule {
- // Selects the methods to which this rule applies.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // The address of the API backend.
- string address = 2;
-
- // The number of seconds to wait for a response from a request. The default
- // deadline for gRPC is infinite (no deadline) and HTTP requests is 5 seconds.
- double deadline = 3;
-
- // Minimum deadline in seconds needed for this method. Calls having deadline
- // value lower than this will be rejected.
- double min_deadline = 4;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/billing.proto b/contrib/libs/googleapis-common-protos/google/api/billing.proto
deleted file mode 100644
index 54c6cca97a..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/billing.proto
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "BillingProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Billing related configuration of the service.
-//
-// The following example shows how to configure monitored resources and metrics
-// for billing:
-//
-// monitored_resources:
-// - type: library.googleapis.com/branch
-// labels:
-// - key: /city
-// description: The city where the library branch is located in.
-// - key: /name
-// description: The name of the branch.
-// metrics:
-// - name: library.googleapis.com/book/borrowed_count
-// metric_kind: DELTA
-// value_type: INT64
-// billing:
-// consumer_destinations:
-// - monitored_resource: library.googleapis.com/branch
-// metrics:
-// - library.googleapis.com/book/borrowed_count
-message Billing {
- // Configuration of a specific billing destination (Currently only support
- // bill against consumer project).
- message BillingDestination {
- // The monitored resource type. The type must be defined in
- // [Service.monitored_resources][google.api.Service.monitored_resources] section.
- string monitored_resource = 1;
-
- // Names of the metrics to report to this billing destination.
- // Each name must be defined in [Service.metrics][google.api.Service.metrics] section.
- repeated string metrics = 2;
- }
-
- // Billing configurations for sending metrics to the consumer project.
- // There can be multiple consumer destinations per service, each one must have
- // a different monitored resource type. A metric can be used in at most
- // one consumer destination.
- repeated BillingDestination consumer_destinations = 8;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/client.proto b/contrib/libs/googleapis-common-protos/google/api/client.proto
deleted file mode 100644
index 2102623d30..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/client.proto
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "ClientProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-extend google.protobuf.MethodOptions {
- // A definition of a client library method signature.
- //
- // In client libraries, each proto RPC corresponds to one or more methods
- // which the end user is able to call, and calls the underlying RPC.
- // Normally, this method receives a single argument (a struct or instance
- // corresponding to the RPC request object). Defining this field will
- // add one or more overloads providing flattened or simpler method signatures
- // in some languages.
- //
- // The fields on the method signature are provided as a comma-separated
- // string.
- //
- // For example, the proto RPC and annotation:
- //
- // rpc CreateSubscription(CreateSubscriptionRequest)
- // returns (Subscription) {
- // option (google.api.method_signature) = "name,topic";
- // }
- //
- // Would add the following Java overload (in addition to the method accepting
- // the request object):
- //
- // public final Subscription createSubscription(String name, String topic)
- //
- // The following backwards-compatibility guidelines apply:
- //
- // * Adding this annotation to an unannotated method is backwards
- // compatible.
- // * Adding this annotation to a method which already has existing
- // method signature annotations is backwards compatible if and only if
- // the new method signature annotation is last in the sequence.
- // * Modifying or removing an existing method signature annotation is
- // a breaking change.
- // * Re-ordering existing method signature annotations is a breaking
- // change.
- repeated string method_signature = 1051;
-}
-
-extend google.protobuf.ServiceOptions {
- // The hostname for this service.
- // This should be specified with no prefix or protocol.
- //
- // Example:
- //
- // service Foo {
- // option (google.api.default_host) = "foo.googleapi.com";
- // ...
- // }
- string default_host = 1049;
-
- // OAuth scopes needed for the client.
- //
- // Example:
- //
- // service Foo {
- // option (google.api.oauth_scopes) = \
- // "https://www.googleapis.com/auth/cloud-platform";
- // ...
- // }
- //
- // If there is more than one scope, use a comma-separated string:
- //
- // Example:
- //
- // service Foo {
- // option (google.api.oauth_scopes) = \
- // "https://www.googleapis.com/auth/cloud-platform,"
- // "https://www.googleapis.com/auth/monitoring";
- // ...
- // }
- string oauth_scopes = 1050;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/config_change.proto b/contrib/libs/googleapis-common-protos/google/api/config_change.proto
deleted file mode 100644
index 1e787931c1..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/config_change.proto
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/configchange;configchange";
-option java_multiple_files = true;
-option java_outer_classname = "ConfigChangeProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Output generated from semantically comparing two versions of a service
-// configuration.
-//
-// Includes detailed information about a field that have changed with
-// applicable advice about potential consequences for the change, such as
-// backwards-incompatibility.
-message ConfigChange {
- // Object hierarchy path to the change, with levels separated by a '.'
- // character. For repeated fields, an applicable unique identifier field is
- // used for the index (usually selector, name, or id). For maps, the term
- // 'key' is used. If the field has no unique identifier, the numeric index
- // is used.
- // Examples:
- // - visibility.rules[selector=="google.LibraryService.CreateBook"].restriction
- // - quota.metric_rules[selector=="google"].metric_costs[key=="reads"].value
- // - logging.producer_destinations[0]
- string element = 1;
-
- // Value of the changed object in the old Service configuration,
- // in JSON format. This field will not be populated if ChangeType == ADDED.
- string old_value = 2;
-
- // Value of the changed object in the new Service configuration,
- // in JSON format. This field will not be populated if ChangeType == REMOVED.
- string new_value = 3;
-
- // The type for this change, either ADDED, REMOVED, or MODIFIED.
- ChangeType change_type = 4;
-
- // Collection of advice provided for this change, useful for determining the
- // possible impact of this change.
- repeated Advice advices = 5;
-}
-
-// Generated advice about this change, used for providing more
-// information about how a change will affect the existing service.
-message Advice {
- // Useful description for why this advice was applied and what actions should
- // be taken to mitigate any implied risks.
- string description = 2;
-}
-
-// Classifies set of possible modifications to an object in the service
-// configuration.
-enum ChangeType {
- // No value was provided.
- CHANGE_TYPE_UNSPECIFIED = 0;
-
- // The changed object exists in the 'new' service configuration, but not
- // in the 'old' service configuration.
- ADDED = 1;
-
- // The changed object exists in the 'old' service configuration, but not
- // in the 'new' service configuration.
- REMOVED = 2;
-
- // The changed object exists in both service configurations, but its value
- // is different.
- MODIFIED = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/consumer.proto b/contrib/libs/googleapis-common-protos/google/api/consumer.proto
deleted file mode 100644
index 4e963168ef..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/consumer.proto
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2016 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "ConsumerProto";
-option java_package = "com.google.api";
-
-
-// A descriptor for defining project properties for a service. One service may
-// have many consumer projects, and the service may want to behave differently
-// depending on some properties on the project. For example, a project may be
-// associated with a school, or a business, or a government agency, a business
-// type property on the project may affect how a service responds to the client.
-// This descriptor defines which properties are allowed to be set on a project.
-//
-// Example:
-//
-// project_properties:
-// properties:
-// - name: NO_WATERMARK
-// type: BOOL
-// description: Allows usage of the API without watermarks.
-// - name: EXTENDED_TILE_CACHE_PERIOD
-// type: INT64
-message ProjectProperties {
- // List of per consumer project-specific properties.
- repeated Property properties = 1;
-}
-
-// Defines project properties.
-//
-// API services can define properties that can be assigned to consumer projects
-// so that backends can perform response customization without having to make
-// additional calls or maintain additional storage. For example, Maps API
-// defines properties that controls map tile cache period, or whether to embed a
-// watermark in a result.
-//
-// These values can be set via API producer console. Only API providers can
-// define and set these properties.
-message Property {
- // Supported data type of the property values
- enum PropertyType {
- // The type is unspecified, and will result in an error.
- UNSPECIFIED = 0;
-
- // The type is `int64`.
- INT64 = 1;
-
- // The type is `bool`.
- BOOL = 2;
-
- // The type is `string`.
- STRING = 3;
-
- // The type is 'double'.
- DOUBLE = 4;
- }
-
- // The name of the property (a.k.a key).
- string name = 1;
-
- // The type of this property.
- PropertyType type = 2;
-
- // The description of the property
- string description = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/context.proto b/contrib/libs/googleapis-common-protos/google/api/context.proto
deleted file mode 100644
index fd03fcae91..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/context.proto
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "ContextProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Context` defines which contexts an API requests.
-//
-// Example:
-//
-// context:
-// rules:
-// - selector: "*"
-// requested:
-// - google.rpc.context.ProjectContext
-// - google.rpc.context.OriginContext
-//
-// The above specifies that all methods in the API request
-// `google.rpc.context.ProjectContext` and
-// `google.rpc.context.OriginContext`.
-//
-// Available context types are defined in package
-// `google.rpc.context`.
-message Context {
- // A list of RPC context rules that apply to individual API methods.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated ContextRule rules = 1;
-}
-
-// A context rule provides information about the context for an individual API
-// element.
-message ContextRule {
- // Selects the methods to which this rule applies.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // A list of full type names of requested contexts.
- repeated string requested = 2;
-
- // A list of full type names of provided contexts.
- repeated string provided = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/control.proto b/contrib/libs/googleapis-common-protos/google/api/control.proto
deleted file mode 100644
index 772d61175f..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/control.proto
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "ControlProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Selects and configures the service controller used by the service. The
-// service controller handles features like abuse, quota, billing, logging,
-// monitoring, etc.
-message Control {
- // The service control environment to use. If empty, no control plane
- // feature (like quota and billing) will be enabled.
- string environment = 1;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/distribution.proto b/contrib/libs/googleapis-common-protos/google/api/distribution.proto
deleted file mode 100644
index f8bb252683..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/distribution.proto
+++ /dev/null
@@ -1,213 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/any.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/distribution;distribution";
-option java_multiple_files = true;
-option java_outer_classname = "DistributionProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Distribution` contains summary statistics for a population of values. It
-// optionally contains a histogram representing the distribution of those values
-// across a set of buckets.
-//
-// The summary statistics are the count, mean, sum of the squared deviation from
-// the mean, the minimum, and the maximum of the set of population of values.
-// The histogram is based on a sequence of buckets and gives a count of values
-// that fall into each bucket. The boundaries of the buckets are given either
-// explicitly or by formulas for buckets of fixed or exponentially increasing
-// widths.
-//
-// Although it is not forbidden, it is generally a bad idea to include
-// non-finite values (infinities or NaNs) in the population of values, as this
-// will render the `mean` and `sum_of_squared_deviation` fields meaningless.
-message Distribution {
- // The range of the population values.
- message Range {
- // The minimum of the population values.
- double min = 1;
-
- // The maximum of the population values.
- double max = 2;
- }
-
- // `BucketOptions` describes the bucket boundaries used to create a histogram
- // for the distribution. The buckets can be in a linear sequence, an
- // exponential sequence, or each bucket can be specified explicitly.
- // `BucketOptions` does not include the number of values in each bucket.
- //
- // A bucket has an inclusive lower bound and exclusive upper bound for the
- // values that are counted for that bucket. The upper bound of a bucket must
- // be strictly greater than the lower bound. The sequence of N buckets for a
- // distribution consists of an underflow bucket (number 0), zero or more
- // finite buckets (number 1 through N - 2) and an overflow bucket (number N -
- // 1). The buckets are contiguous: the lower bound of bucket i (i > 0) is the
- // same as the upper bound of bucket i - 1. The buckets span the whole range
- // of finite values: lower bound of the underflow bucket is -infinity and the
- // upper bound of the overflow bucket is +infinity. The finite buckets are
- // so-called because both bounds are finite.
- message BucketOptions {
- // Specifies a linear sequence of buckets that all have the same width
- // (except overflow and underflow). Each bucket represents a constant
- // absolute uncertainty on the specific value in the bucket.
- //
- // There are `num_finite_buckets + 2` (= N) buckets. Bucket `i` has the
- // following boundaries:
- //
- // Upper bound (0 <= i < N-1): offset + (width * i).
- // Lower bound (1 <= i < N): offset + (width * (i - 1)).
- message Linear {
- // Must be greater than 0.
- int32 num_finite_buckets = 1;
-
- // Must be greater than 0.
- double width = 2;
-
- // Lower bound of the first bucket.
- double offset = 3;
- }
-
- // Specifies an exponential sequence of buckets that have a width that is
- // proportional to the value of the lower bound. Each bucket represents a
- // constant relative uncertainty on a specific value in the bucket.
- //
- // There are `num_finite_buckets + 2` (= N) buckets. Bucket `i` has the
- // following boundaries:
- //
- // Upper bound (0 <= i < N-1): scale * (growth_factor ^ i).
- // Lower bound (1 <= i < N): scale * (growth_factor ^ (i - 1)).
- message Exponential {
- // Must be greater than 0.
- int32 num_finite_buckets = 1;
-
- // Must be greater than 1.
- double growth_factor = 2;
-
- // Must be greater than 0.
- double scale = 3;
- }
-
- // Specifies a set of buckets with arbitrary widths.
- //
- // There are `size(bounds) + 1` (= N) buckets. Bucket `i` has the following
- // boundaries:
- //
- // Upper bound (0 <= i < N-1): bounds[i]
- // Lower bound (1 <= i < N); bounds[i - 1]
- //
- // The `bounds` field must contain at least one element. If `bounds` has
- // only one element, then there are no finite buckets, and that single
- // element is the common boundary of the overflow and underflow buckets.
- message Explicit {
- // The values must be monotonically increasing.
- repeated double bounds = 1;
- }
-
- // Exactly one of these three fields must be set.
- oneof options {
- // The linear bucket.
- Linear linear_buckets = 1;
-
- // The exponential buckets.
- Exponential exponential_buckets = 2;
-
- // The explicit buckets.
- Explicit explicit_buckets = 3;
- }
- }
-
- // Exemplars are example points that may be used to annotate aggregated
- // distribution values. They are metadata that gives information about a
- // particular value added to a Distribution bucket, such as a trace ID that
- // was active when a value was added. They may contain further information,
- // such as a example values and timestamps, origin, etc.
- message Exemplar {
- // Value of the exemplar point. This value determines to which bucket the
- // exemplar belongs.
- double value = 1;
-
- // The observation (sampling) time of the above value.
- google.protobuf.Timestamp timestamp = 2;
-
- // Contextual information about the example value. Examples are:
- //
- // Trace ID: type.googleapis.com/google.devtools.cloudtrace.v1.Trace
- //
- // Literal string: type.googleapis.com/google.protobuf.StringValue
- //
- // Labels dropped during aggregation:
- // type.googleapis.com/google.monitoring.v3.DroppedLabels
- //
- // There may be only a single attachment of any given message type in a
- // single exemplar, and this is enforced by the system.
- repeated google.protobuf.Any attachments = 3;
- }
-
- // The number of values in the population. Must be non-negative. This value
- // must equal the sum of the values in `bucket_counts` if a histogram is
- // provided.
- int64 count = 1;
-
- // The arithmetic mean of the values in the population. If `count` is zero
- // then this field must be zero.
- double mean = 2;
-
- // The sum of squared deviations from the mean of the values in the
- // population. For values x_i this is:
- //
- // Sum[i=1..n]((x_i - mean)^2)
- //
- // Knuth, "The Art of Computer Programming", Vol. 2, page 323, 3rd edition
- // describes Welford's method for accumulating this sum in one pass.
- //
- // If `count` is zero then this field must be zero.
- double sum_of_squared_deviation = 3;
-
- // If specified, contains the range of the population values. The field
- // must not be present if the `count` is zero.
- Range range = 4;
-
- // Defines the histogram bucket boundaries. If the distribution does not
- // contain a histogram, then omit this field.
- BucketOptions bucket_options = 6;
-
- // The number of values in each bucket of the histogram, as described in
- // `bucket_options`. If the distribution does not have a histogram, then omit
- // this field. If there is a histogram, then the sum of the values in
- // `bucket_counts` must equal the value in the `count` field of the
- // distribution.
- //
- // If present, `bucket_counts` should contain N values, where N is the number
- // of buckets specified in `bucket_options`. If you supply fewer than N
- // values, the remaining values are assumed to be 0.
- //
- // The order of the values in `bucket_counts` follows the bucket numbering
- // schemes described for the three bucket types. The first value must be the
- // count for the underflow bucket (number 0). The next N-2 values are the
- // counts for the finite buckets (number 1 through N-2). The N'th value in
- // `bucket_counts` is the count for the overflow bucket (number N-1).
- repeated int64 bucket_counts = 7;
-
- // Must be in increasing order of `value` field.
- repeated Exemplar exemplars = 10;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/documentation.proto b/contrib/libs/googleapis-common-protos/google/api/documentation.proto
deleted file mode 100644
index 03cd73109a..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/documentation.proto
+++ /dev/null
@@ -1,157 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "DocumentationProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Documentation` provides the information for describing a service.
-//
-// Example:
-// <pre><code>documentation:
-// summary: >
-// The Google Calendar API gives access
-// to most calendar features.
-// pages:
-// - name: Overview
-// content: &#40;== include google/foo/overview.md ==&#41;
-// - name: Tutorial
-// content: &#40;== include google/foo/tutorial.md ==&#41;
-// subpages;
-// - name: Java
-// content: &#40;== include google/foo/tutorial_java.md ==&#41;
-// rules:
-// - selector: google.calendar.Calendar.Get
-// description: >
-// ...
-// - selector: google.calendar.Calendar.Put
-// description: >
-// ...
-// </code></pre>
-// Documentation is provided in markdown syntax. In addition to
-// standard markdown features, definition lists, tables and fenced
-// code blocks are supported. Section headers can be provided and are
-// interpreted relative to the section nesting of the context where
-// a documentation fragment is embedded.
-//
-// Documentation from the IDL is merged with documentation defined
-// via the config at normalization time, where documentation provided
-// by config rules overrides IDL provided.
-//
-// A number of constructs specific to the API platform are supported
-// in documentation text.
-//
-// In order to reference a proto element, the following
-// notation can be used:
-// <pre><code>&#91;fully.qualified.proto.name]&#91;]</code></pre>
-// To override the display text used for the link, this can be used:
-// <pre><code>&#91;display text]&#91;fully.qualified.proto.name]</code></pre>
-// Text can be excluded from doc using the following notation:
-// <pre><code>&#40;-- internal comment --&#41;</code></pre>
-//
-// A few directives are available in documentation. Note that
-// directives must appear on a single line to be properly
-// identified. The `include` directive includes a markdown file from
-// an external source:
-// <pre><code>&#40;== include path/to/file ==&#41;</code></pre>
-// The `resource_for` directive marks a message to be the resource of
-// a collection in REST view. If it is not specified, tools attempt
-// to infer the resource from the operations in a collection:
-// <pre><code>&#40;== resource_for v1.shelves.books ==&#41;</code></pre>
-// The directive `suppress_warning` does not directly affect documentation
-// and is documented together with service config validation.
-message Documentation {
- // A short summary of what the service does. Can only be provided by
- // plain text.
- string summary = 1;
-
- // The top level pages for the documentation set.
- repeated Page pages = 5;
-
- // A list of documentation rules that apply to individual API elements.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated DocumentationRule rules = 3;
-
- // The URL to the root of documentation.
- string documentation_root_url = 4;
-
- // Declares a single overview page. For example:
- // <pre><code>documentation:
- // summary: ...
- // overview: &#40;== include overview.md ==&#41;
- // </code></pre>
- // This is a shortcut for the following declaration (using pages style):
- // <pre><code>documentation:
- // summary: ...
- // pages:
- // - name: Overview
- // content: &#40;== include overview.md ==&#41;
- // </code></pre>
- // Note: you cannot specify both `overview` field and `pages` field.
- string overview = 2;
-}
-
-// A documentation rule provides information about individual API elements.
-message DocumentationRule {
- // The selector is a comma-separated list of patterns. Each pattern is a
- // qualified name of the element which may end in "*", indicating a wildcard.
- // Wildcards are only allowed at the end and for a whole component of the
- // qualified name, i.e. "foo.*" is ok, but not "foo.b*" or "foo.*.bar". To
- // specify a default for all applicable elements, the whole pattern "*"
- // is used.
- string selector = 1;
-
- // Description of the selected API(s).
- string description = 2;
-
- // Deprecation description of the selected element(s). It can be provided if an
- // element is marked as `deprecated`.
- string deprecation_description = 3;
-}
-
-// Represents a documentation page. A page can contain subpages to represent
-// nested documentation set structure.
-message Page {
- // The name of the page. It will be used as an identity of the page to
- // generate URI of the page, text of the link to this page in navigation,
- // etc. The full page name (start from the root page name to this page
- // concatenated with `.`) can be used as reference to the page in your
- // documentation. For example:
- // <pre><code>pages:
- // - name: Tutorial
- // content: &#40;== include tutorial.md ==&#41;
- // subpages:
- // - name: Java
- // content: &#40;== include tutorial_java.md ==&#41;
- // </code></pre>
- // You can reference `Java` page using Markdown reference link syntax:
- // `[Java][Tutorial.Java]`.
- string name = 1;
-
- // The Markdown content of the page. You can use <code>&#40;== include {path} ==&#41;</code>
- // to include content from a Markdown file.
- string content = 2;
-
- // Subpages of this page. The order of subpages specified here will be
- // honored in the generated docset.
- repeated Page subpages = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/endpoint.proto b/contrib/libs/googleapis-common-protos/google/api/endpoint.proto
deleted file mode 100644
index c7bb13b037..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/endpoint.proto
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "EndpointProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Endpoint` describes a network endpoint that serves a set of APIs.
-// A service may expose any number of endpoints, and all endpoints share the
-// same service configuration, such as quota configuration and monitoring
-// configuration.
-//
-// Example service configuration:
-//
-// name: library-example.googleapis.com
-// endpoints:
-// # Below entry makes 'google.example.library.v1.Library'
-// # API be served from endpoint address library-example.googleapis.com.
-// # It also allows HTTP OPTIONS calls to be passed to the backend, for
-// # it to decide whether the subsequent cross-origin request is
-// # allowed to proceed.
-// - name: library-example.googleapis.com
-// allow_cors: true
-message Endpoint {
- // The canonical name of this endpoint.
- string name = 1;
-
- // DEPRECATED: This field is no longer supported. Instead of using aliases,
- // please specify multiple [google.api.Endpoint][google.api.Endpoint] for each of the intended
- // aliases.
- //
- // Additional names that this endpoint will be hosted on.
- repeated string aliases = 2;
-
- // The list of features enabled on this endpoint.
- repeated string features = 4;
-
- // The specification of an Internet routable address of API frontend that will
- // handle requests to this [API Endpoint](https://cloud.google.com/apis/design/glossary).
- // It should be either a valid IPv4 address or a fully-qualified domain name.
- // For example, "8.8.8.8" or "myservice.appspot.com".
- string target = 101;
-
- // Allowing
- // [CORS](https://en.wikipedia.org/wiki/Cross-origin_resource_sharing), aka
- // cross-domain traffic, would allow the backends served from this endpoint to
- // receive and respond to HTTP OPTIONS requests. The response will be used by
- // the browser to determine whether the subsequent cross-origin request is
- // allowed to proceed.
- bool allow_cors = 5;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/field_behavior.proto b/contrib/libs/googleapis-common-protos/google/api/field_behavior.proto
deleted file mode 100644
index 686667954a..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/field_behavior.proto
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "FieldBehaviorProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-extend google.protobuf.FieldOptions {
- // A designation of a specific field behavior (required, output only, etc.)
- // in protobuf messages.
- //
- // Examples:
- //
- // string name = 1 [(google.api.field_behavior) = REQUIRED];
- // State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
- // google.protobuf.Duration ttl = 1
- // [(google.api.field_behavior) = INPUT_ONLY];
- // google.protobuf.Timestamp expire_time = 1
- // [(google.api.field_behavior) = OUTPUT_ONLY,
- // (google.api.field_behavior) = IMMUTABLE];
- repeated google.api.FieldBehavior field_behavior = 1052;
-}
-
-// An indicator of the behavior of a given field (for example, that a field
-// is required in requests, or given as output but ignored as input).
-// This **does not** change the behavior in protocol buffers itself; it only
-// denotes the behavior and may affect how API tooling handles the field.
-//
-// Note: This enum **may** receive new values in the future.
-enum FieldBehavior {
- // Conventional default for enums. Do not use this.
- FIELD_BEHAVIOR_UNSPECIFIED = 0;
-
- // Specifically denotes a field as optional.
- // While all fields in protocol buffers are optional, this may be specified
- // for emphasis if appropriate.
- OPTIONAL = 1;
-
- // Denotes a field as required.
- // This indicates that the field **must** be provided as part of the request,
- // and failure to do so will cause an error (usually `INVALID_ARGUMENT`).
- REQUIRED = 2;
-
- // Denotes a field as output only.
- // This indicates that the field is provided in responses, but including the
- // field in a request does nothing (the server *must* ignore it and
- // *must not* throw an error as a result of the field's presence).
- OUTPUT_ONLY = 3;
-
- // Denotes a field as input only.
- // This indicates that the field is provided in requests, and the
- // corresponding field is not included in output.
- INPUT_ONLY = 4;
-
- // Denotes a field as immutable.
- // This indicates that the field may be set once in a request to create a
- // resource, but may not be changed thereafter.
- IMMUTABLE = 5;
-
- // Denotes that a (repeated) field is an unordered list.
- // This indicates that the service may provide the elements of the list
- // in any arbitrary order, rather than the order the user originally
- // provided. Additionally, the list's order may or may not be stable.
- UNORDERED_LIST = 6;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/http.proto b/contrib/libs/googleapis-common-protos/google/api/http.proto
deleted file mode 100644
index 2bd3a19bfa..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/http.proto
+++ /dev/null
@@ -1,318 +0,0 @@
-// Copyright 2018 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "HttpProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Defines the HTTP configuration for an API service. It contains a list of
-// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method
-// to one or more HTTP REST API methods.
-message Http {
- // A list of HTTP configuration rules that apply to individual API methods.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated HttpRule rules = 1;
-
- // When set to true, URL path parmeters will be fully URI-decoded except in
- // cases of single segment matches in reserved expansion, where "%2F" will be
- // left encoded.
- //
- // The default behavior is to not decode RFC 6570 reserved characters in multi
- // segment matches.
- bool fully_decode_reserved_expansion = 2;
-}
-
-// `HttpRule` defines the mapping of an RPC method to one or more HTTP
-// REST API methods. The mapping specifies how different portions of the RPC
-// request message are mapped to URL path, URL query parameters, and
-// HTTP request body. The mapping is typically specified as an
-// `google.api.http` annotation on the RPC method,
-// see "google/api/annotations.proto" for details.
-//
-// The mapping consists of a field specifying the path template and
-// method kind. The path template can refer to fields in the request
-// message, as in the example below which describes a REST GET
-// operation on a resource collection of messages:
-//
-//
-// service Messaging {
-// rpc GetMessage(GetMessageRequest) returns (Message) {
-// option (google.api.http).get = "/v1/messages/{message_id}/{sub.subfield}";
-// }
-// }
-// message GetMessageRequest {
-// message SubMessage {
-// string subfield = 1;
-// }
-// string message_id = 1; // mapped to the URL
-// SubMessage sub = 2; // `sub.subfield` is url-mapped
-// }
-// message Message {
-// string text = 1; // content of the resource
-// }
-//
-// The same http annotation can alternatively be expressed inside the
-// `GRPC API Configuration` YAML file.
-//
-// http:
-// rules:
-// - selector: <proto_package_name>.Messaging.GetMessage
-// get: /v1/messages/{message_id}/{sub.subfield}
-//
-// This definition enables an automatic, bidrectional mapping of HTTP
-// JSON to RPC. Example:
-//
-// HTTP | RPC
-// -----|-----
-// `GET /v1/messages/123456/foo` | `GetMessage(message_id: "123456" sub: SubMessage(subfield: "foo"))`
-//
-// In general, not only fields but also field paths can be referenced
-// from a path pattern. Fields mapped to the path pattern cannot be
-// repeated and must have a primitive (non-message) type.
-//
-// Any fields in the request message which are not bound by the path
-// pattern automatically become (optional) HTTP query
-// parameters. Assume the following definition of the request message:
-//
-//
-// service Messaging {
-// rpc GetMessage(GetMessageRequest) returns (Message) {
-// option (google.api.http).get = "/v1/messages/{message_id}";
-// }
-// }
-// message GetMessageRequest {
-// message SubMessage {
-// string subfield = 1;
-// }
-// string message_id = 1; // mapped to the URL
-// int64 revision = 2; // becomes a parameter
-// SubMessage sub = 3; // `sub.subfield` becomes a parameter
-// }
-//
-//
-// This enables a HTTP JSON to RPC mapping as below:
-//
-// HTTP | RPC
-// -----|-----
-// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: "foo"))`
-//
-// Note that fields which are mapped to HTTP parameters must have a
-// primitive type or a repeated primitive type. Message types are not
-// allowed. In the case of a repeated type, the parameter can be
-// repeated in the URL, as in `...?param=A&param=B`.
-//
-// For HTTP method kinds which allow a request body, the `body` field
-// specifies the mapping. Consider a REST update method on the
-// message resource collection:
-//
-//
-// service Messaging {
-// rpc UpdateMessage(UpdateMessageRequest) returns (Message) {
-// option (google.api.http) = {
-// put: "/v1/messages/{message_id}"
-// body: "message"
-// };
-// }
-// }
-// message UpdateMessageRequest {
-// string message_id = 1; // mapped to the URL
-// Message message = 2; // mapped to the body
-// }
-//
-//
-// The following HTTP JSON to RPC mapping is enabled, where the
-// representation of the JSON in the request body is determined by
-// protos JSON encoding:
-//
-// HTTP | RPC
-// -----|-----
-// `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: "123456" message { text: "Hi!" })`
-//
-// The special name `*` can be used in the body mapping to define that
-// every field not bound by the path template should be mapped to the
-// request body. This enables the following alternative definition of
-// the update method:
-//
-// service Messaging {
-// rpc UpdateMessage(Message) returns (Message) {
-// option (google.api.http) = {
-// put: "/v1/messages/{message_id}"
-// body: "*"
-// };
-// }
-// }
-// message Message {
-// string message_id = 1;
-// string text = 2;
-// }
-//
-//
-// The following HTTP JSON to RPC mapping is enabled:
-//
-// HTTP | RPC
-// -----|-----
-// `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: "123456" text: "Hi!")`
-//
-// Note that when using `*` in the body mapping, it is not possible to
-// have HTTP parameters, as all fields not bound by the path end in
-// the body. This makes this option more rarely used in practice of
-// defining REST APIs. The common usage of `*` is in custom methods
-// which don't use the URL at all for transferring data.
-//
-// It is possible to define multiple HTTP methods for one RPC by using
-// the `additional_bindings` option. Example:
-//
-// service Messaging {
-// rpc GetMessage(GetMessageRequest) returns (Message) {
-// option (google.api.http) = {
-// get: "/v1/messages/{message_id}"
-// additional_bindings {
-// get: "/v1/users/{user_id}/messages/{message_id}"
-// }
-// };
-// }
-// }
-// message GetMessageRequest {
-// string message_id = 1;
-// string user_id = 2;
-// }
-//
-//
-// This enables the following two alternative HTTP JSON to RPC
-// mappings:
-//
-// HTTP | RPC
-// -----|-----
-// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")`
-// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: "123456")`
-//
-// # Rules for HTTP mapping
-//
-// The rules for mapping HTTP path, query parameters, and body fields
-// to the request message are as follows:
-//
-// 1. The `body` field specifies either `*` or a field path, or is
-// omitted. If omitted, it indicates there is no HTTP request body.
-// 2. Leaf fields (recursive expansion of nested messages in the
-// request) can be classified into three types:
-// (a) Matched in the URL template.
-// (b) Covered by body (if body is `*`, everything except (a) fields;
-// else everything under the body field)
-// (c) All other fields.
-// 3. URL query parameters found in the HTTP request are mapped to (c) fields.
-// 4. Any body sent with an HTTP request can contain only (b) fields.
-//
-// The syntax of the path template is as follows:
-//
-// Template = "/" Segments [ Verb ] ;
-// Segments = Segment { "/" Segment } ;
-// Segment = "*" | "**" | LITERAL | Variable ;
-// Variable = "{" FieldPath [ "=" Segments ] "}" ;
-// FieldPath = IDENT { "." IDENT } ;
-// Verb = ":" LITERAL ;
-//
-// The syntax `*` matches a single path segment. The syntax `**` matches zero
-// or more path segments, which must be the last part of the path except the
-// `Verb`. The syntax `LITERAL` matches literal text in the path.
-//
-// The syntax `Variable` matches part of the URL path as specified by its
-// template. A variable template must not contain other variables. If a variable
-// matches a single path segment, its template may be omitted, e.g. `{var}`
-// is equivalent to `{var=*}`.
-//
-// If a variable contains exactly one path segment, such as `"{var}"` or
-// `"{var=*}"`, when such a variable is expanded into a URL path, all characters
-// except `[-_.~0-9a-zA-Z]` are percent-encoded. Such variables show up in the
-// Discovery Document as `{var}`.
-//
-// If a variable contains one or more path segments, such as `"{var=foo/*}"`
-// or `"{var=**}"`, when such a variable is expanded into a URL path, all
-// characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. Such variables
-// show up in the Discovery Document as `{+var}`.
-//
-// NOTE: While the single segment variable matches the semantics of
-// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2
-// Simple String Expansion, the multi segment variable **does not** match
-// RFC 6570 Reserved Expansion. The reason is that the Reserved Expansion
-// does not expand special characters like `?` and `#`, which would lead
-// to invalid URLs.
-//
-// NOTE: the field paths in variables and in the `body` must not refer to
-// repeated fields or map fields.
-message HttpRule {
- // Selects methods to which this rule applies.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // Determines the URL pattern is matched by this rules. This pattern can be
- // used with any of the {get|put|post|delete|patch} methods. A custom method
- // can be defined using the 'custom' field.
- oneof pattern {
- // Used for listing and getting information about resources.
- string get = 2;
-
- // Used for updating a resource.
- string put = 3;
-
- // Used for creating a resource.
- string post = 4;
-
- // Used for deleting a resource.
- string delete = 5;
-
- // Used for updating a resource.
- string patch = 6;
-
- // The custom pattern is used for specifying an HTTP method that is not
- // included in the `pattern` field, such as HEAD, or "*" to leave the
- // HTTP method unspecified for this rule. The wild-card rule is useful
- // for services that provide content to Web (HTML) clients.
- CustomHttpPattern custom = 8;
- }
-
- // The name of the request field whose value is mapped to the HTTP body, or
- // `*` for mapping all fields not captured by the path pattern to the HTTP
- // body. NOTE: the referred field must not be a repeated field and must be
- // present at the top-level of request message type.
- string body = 7;
-
- // Optional. The name of the response field whose value is mapped to the HTTP
- // body of response. Other response fields are ignored. When
- // not set, the response message will be used as HTTP body of response.
- string response_body = 12;
-
- // Additional HTTP bindings for the selector. Nested bindings must
- // not contain an `additional_bindings` field themselves (that is,
- // the nesting may only be one level deep).
- repeated HttpRule additional_bindings = 11;
-}
-
-// A custom pattern is used for defining custom HTTP verb.
-message CustomHttpPattern {
- // The name of this custom HTTP verb.
- string kind = 1;
-
- // The path matched by this custom verb.
- string path = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/httpbody.proto b/contrib/libs/googleapis-common-protos/google/api/httpbody.proto
deleted file mode 100644
index 0e86e267d7..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/httpbody.proto
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/any.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/httpbody;httpbody";
-option java_multiple_files = true;
-option java_outer_classname = "HttpBodyProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Message that represents an arbitrary HTTP body. It should only be used for
-// payload formats that can't be represented as JSON, such as raw binary or
-// an HTML page.
-//
-//
-// This message can be used both in streaming and non-streaming API methods in
-// the request as well as the response.
-//
-// It can be used as a top-level request field, which is convenient if one
-// wants to extract parameters from either the URL or HTTP template into the
-// request fields and also want access to the raw HTTP body.
-//
-// Example:
-//
-// message GetResourceRequest {
-// // A unique request id.
-// string request_id = 1;
-//
-// // The raw HTTP body is bound to this field.
-// google.api.HttpBody http_body = 2;
-// }
-//
-// service ResourceService {
-// rpc GetResource(GetResourceRequest) returns (google.api.HttpBody);
-// rpc UpdateResource(google.api.HttpBody) returns (google.protobuf.Empty);
-// }
-//
-// Example with streaming methods:
-//
-// service CaldavService {
-// rpc GetCalendar(stream google.api.HttpBody)
-// returns (stream google.api.HttpBody);
-// rpc UpdateCalendar(stream google.api.HttpBody)
-// returns (stream google.api.HttpBody);
-// }
-//
-// Use of this type only changes how the request and response bodies are
-// handled, all other features will continue to work unchanged.
-message HttpBody {
- // The HTTP Content-Type string representing the content type of the body.
- string content_type = 1;
-
- // HTTP body binary data.
- bytes data = 2;
-
- // Application specific response metadata. Must be set in the first response
- // for streaming APIs.
- repeated google.protobuf.Any extensions = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/label.proto b/contrib/libs/googleapis-common-protos/google/api/label.proto
deleted file mode 100644
index 3c20752ceb..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/label.proto
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/api/label;label";
-option java_multiple_files = true;
-option java_outer_classname = "LabelProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// A description of a label.
-message LabelDescriptor {
- // Value types that can be used as label values.
- enum ValueType {
- // A variable-length string. This is the default.
- STRING = 0;
-
- // Boolean; true or false.
- BOOL = 1;
-
- // A 64-bit signed integer.
- INT64 = 2;
- }
-
- // The label key.
- string key = 1;
-
- // The type of data that can be assigned to the label.
- ValueType value_type = 2;
-
- // A human-readable description for the label.
- string description = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/launch_stage.proto b/contrib/libs/googleapis-common-protos/google/api/launch_stage.proto
deleted file mode 100644
index 55fd914242..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/launch_stage.proto
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api;api";
-option java_multiple_files = true;
-option java_outer_classname = "LaunchStageProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-// The launch stage as defined by [Google Cloud Platform
-// Launch Stages](http://cloud.google.com/terms/launch-stages).
-enum LaunchStage {
- // Do not use this default value.
- LAUNCH_STAGE_UNSPECIFIED = 0;
-
- // Early Access features are limited to a closed group of testers. To use
- // these features, you must sign up in advance and sign a Trusted Tester
- // agreement (which includes confidentiality provisions). These features may
- // be unstable, changed in backward-incompatible ways, and are not
- // guaranteed to be released.
- EARLY_ACCESS = 1;
-
- // Alpha is a limited availability test for releases before they are cleared
- // for widespread use. By Alpha, all significant design issues are resolved
- // and we are in the process of verifying functionality. Alpha customers
- // need to apply for access, agree to applicable terms, and have their
- // projects whitelisted. Alpha releases don’t have to be feature complete,
- // no SLAs are provided, and there are no technical support obligations, but
- // they will be far enough along that customers can actually use them in
- // test environments or for limited-use tests -- just like they would in
- // normal production cases.
- ALPHA = 2;
-
- // Beta is the point at which we are ready to open a release for any
- // customer to use. There are no SLA or technical support obligations in a
- // Beta release. Products will be complete from a feature perspective, but
- // may have some open outstanding issues. Beta releases are suitable for
- // limited production use cases.
- BETA = 3;
-
- // GA features are open to all developers and are considered stable and
- // fully qualified for production use.
- GA = 4;
-
- // Deprecated features are scheduled to be shut down and removed. For more
- // information, see the “Deprecation Policy” section of our [Terms of
- // Service](https://cloud.google.com/terms/)
- // and the [Google Cloud Platform Subject to the Deprecation
- // Policy](https://cloud.google.com/terms/deprecation) documentation.
- DEPRECATED = 5;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/log.proto b/contrib/libs/googleapis-common-protos/google/api/log.proto
deleted file mode 100644
index 673b5245ab..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/log.proto
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/label.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "LogProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// A description of a log type. Example in YAML format:
-//
-// - name: library.googleapis.com/activity_history
-// description: The history of borrowing and returning library items.
-// display_name: Activity
-// labels:
-// - key: /customer_id
-// description: Identifier of a library customer
-message LogDescriptor {
- // The name of the log. It must be less than 512 characters long and can
- // include the following characters: upper- and lower-case alphanumeric
- // characters [A-Za-z0-9], and punctuation characters including
- // slash, underscore, hyphen, period [/_-.].
- string name = 1;
-
- // The set of labels that are available to describe a specific log entry.
- // Runtime requests that contain labels not specified here are
- // considered invalid.
- repeated LabelDescriptor labels = 2;
-
- // A human-readable description of this log. This information appears in
- // the documentation and can contain details.
- string description = 3;
-
- // The human-readable name for this log. This information appears on
- // the user interface and should be concise.
- string display_name = 4;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/logging.proto b/contrib/libs/googleapis-common-protos/google/api/logging.proto
deleted file mode 100644
index ee7c1966e4..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/logging.proto
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "LoggingProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Logging configuration of the service.
-//
-// The following example shows how to configure logs to be sent to the
-// producer and consumer projects. In the example, the `activity_history`
-// log is sent to both the producer and consumer projects, whereas the
-// `purchase_history` log is only sent to the producer project.
-//
-// monitored_resources:
-// - type: library.googleapis.com/branch
-// labels:
-// - key: /city
-// description: The city where the library branch is located in.
-// - key: /name
-// description: The name of the branch.
-// logs:
-// - name: activity_history
-// labels:
-// - key: /customer_id
-// - name: purchase_history
-// logging:
-// producer_destinations:
-// - monitored_resource: library.googleapis.com/branch
-// logs:
-// - activity_history
-// - purchase_history
-// consumer_destinations:
-// - monitored_resource: library.googleapis.com/branch
-// logs:
-// - activity_history
-message Logging {
- // Configuration of a specific logging destination (the producer project
- // or the consumer project).
- message LoggingDestination {
- // The monitored resource type. The type must be defined in the
- // [Service.monitored_resources][google.api.Service.monitored_resources] section.
- string monitored_resource = 3;
-
- // Names of the logs to be sent to this destination. Each name must
- // be defined in the [Service.logs][google.api.Service.logs] section. If the log name is
- // not a domain scoped name, it will be automatically prefixed with
- // the service name followed by "/".
- repeated string logs = 1;
- }
-
- // Logging configurations for sending logs to the producer project.
- // There can be multiple producer destinations, each one must have a
- // different monitored resource type. A log can be used in at most
- // one producer destination.
- repeated LoggingDestination producer_destinations = 1;
-
- // Logging configurations for sending logs to the consumer project.
- // There can be multiple consumer destinations, each one must have a
- // different monitored resource type. A log can be used in at most
- // one consumer destination.
- repeated LoggingDestination consumer_destinations = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/metric.proto b/contrib/libs/googleapis-common-protos/google/api/metric.proto
deleted file mode 100644
index 5473ce5bfe..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/metric.proto
+++ /dev/null
@@ -1,192 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/label.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/metric;metric";
-option java_multiple_files = true;
-option java_outer_classname = "MetricProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Defines a metric type and its schema. Once a metric descriptor is created,
-// deleting or altering it stops data collection and makes the metric type's
-// existing data unusable.
-message MetricDescriptor {
- // The kind of measurement. It describes how the data is reported.
- enum MetricKind {
- // Do not use this default value.
- METRIC_KIND_UNSPECIFIED = 0;
-
- // An instantaneous measurement of a value.
- GAUGE = 1;
-
- // The change in a value during a time interval.
- DELTA = 2;
-
- // A value accumulated over a time interval. Cumulative
- // measurements in a time series should have the same start time
- // and increasing end times, until an event resets the cumulative
- // value to zero and sets a new start time for the following
- // points.
- CUMULATIVE = 3;
- }
-
- // The value type of a metric.
- enum ValueType {
- // Do not use this default value.
- VALUE_TYPE_UNSPECIFIED = 0;
-
- // The value is a boolean.
- // This value type can be used only if the metric kind is `GAUGE`.
- BOOL = 1;
-
- // The value is a signed 64-bit integer.
- INT64 = 2;
-
- // The value is a double precision floating point number.
- DOUBLE = 3;
-
- // The value is a text string.
- // This value type can be used only if the metric kind is `GAUGE`.
- STRING = 4;
-
- // The value is a [`Distribution`][google.api.Distribution].
- DISTRIBUTION = 5;
-
- // The value is money.
- MONEY = 6;
- }
-
- // The resource name of the metric descriptor.
- string name = 1;
-
- // The metric type, including its DNS name prefix. The type is not
- // URL-encoded. All user-defined custom metric types have the DNS name
- // `custom.googleapis.com`. Metric types should use a natural hierarchical
- // grouping. For example:
- //
- // "custom.googleapis.com/invoice/paid/amount"
- // "appengine.googleapis.com/http/server/response_latencies"
- string type = 8;
-
- // The set of labels that can be used to describe a specific
- // instance of this metric type. For example, the
- // `appengine.googleapis.com/http/server/response_latencies` metric
- // type has a label for the HTTP response code, `response_code`, so
- // you can look at latencies for successful responses or just
- // for responses that failed.
- repeated LabelDescriptor labels = 2;
-
- // Whether the metric records instantaneous values, changes to a value, etc.
- // Some combinations of `metric_kind` and `value_type` might not be supported.
- MetricKind metric_kind = 3;
-
- // Whether the measurement is an integer, a floating-point number, etc.
- // Some combinations of `metric_kind` and `value_type` might not be supported.
- ValueType value_type = 4;
-
- // The unit in which the metric value is reported. It is only applicable
- // if the `value_type` is `INT64`, `DOUBLE`, or `DISTRIBUTION`. The
- // supported units are a subset of [The Unified Code for Units of
- // Measure](http://unitsofmeasure.org/ucum.html) standard:
- //
- // **Basic units (UNIT)**
- //
- // * `bit` bit
- // * `By` byte
- // * `s` second
- // * `min` minute
- // * `h` hour
- // * `d` day
- //
- // **Prefixes (PREFIX)**
- //
- // * `k` kilo (10**3)
- // * `M` mega (10**6)
- // * `G` giga (10**9)
- // * `T` tera (10**12)
- // * `P` peta (10**15)
- // * `E` exa (10**18)
- // * `Z` zetta (10**21)
- // * `Y` yotta (10**24)
- // * `m` milli (10**-3)
- // * `u` micro (10**-6)
- // * `n` nano (10**-9)
- // * `p` pico (10**-12)
- // * `f` femto (10**-15)
- // * `a` atto (10**-18)
- // * `z` zepto (10**-21)
- // * `y` yocto (10**-24)
- // * `Ki` kibi (2**10)
- // * `Mi` mebi (2**20)
- // * `Gi` gibi (2**30)
- // * `Ti` tebi (2**40)
- //
- // **Grammar**
- //
- // The grammar also includes these connectors:
- //
- // * `/` division (as an infix operator, e.g. `1/s`).
- // * `.` multiplication (as an infix operator, e.g. `GBy.d`)
- //
- // The grammar for a unit is as follows:
- //
- // Expression = Component { "." Component } { "/" Component } ;
- //
- // Component = ( [ PREFIX ] UNIT | "%" ) [ Annotation ]
- // | Annotation
- // | "1"
- // ;
- //
- // Annotation = "{" NAME "}" ;
- //
- // Notes:
- //
- // * `Annotation` is just a comment if it follows a `UNIT` and is
- // equivalent to `1` if it is used alone. For examples,
- // `{requests}/s == 1/s`, `By{transmitted}/s == By/s`.
- // * `NAME` is a sequence of non-blank printable ASCII characters not
- // containing '{' or '}'.
- // * `1` represents dimensionless value 1, such as in `1/s`.
- // * `%` represents dimensionless value 1/100, and annotates values giving
- // a percentage.
- string unit = 5;
-
- // A detailed description of the metric, which can be used in documentation.
- string description = 6;
-
- // A concise name for the metric, which can be displayed in user interfaces.
- // Use sentence case without an ending period, for example "Request count".
- // This field is optional but it is recommended to be set for any metrics
- // associated with user-visible concepts, such as Quota.
- string display_name = 7;
-}
-
-// A specific metric, identified by specifying values for all of the
-// labels of a [`MetricDescriptor`][google.api.MetricDescriptor].
-message Metric {
- // An existing metric type, see [google.api.MetricDescriptor][google.api.MetricDescriptor].
- // For example, `custom.googleapis.com/invoice/paid/amount`.
- string type = 3;
-
- // The set of label values that uniquely identify this metric. All
- // labels listed in the `MetricDescriptor` must be assigned values.
- map<string, string> labels = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/monitored_resource.proto b/contrib/libs/googleapis-common-protos/google/api/monitored_resource.proto
deleted file mode 100644
index 189ccd4aa7..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/monitored_resource.proto
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/label.proto";
-import "google/protobuf/struct.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/api/monitoredres;monitoredres";
-option java_multiple_files = true;
-option java_outer_classname = "MonitoredResourceProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// An object that describes the schema of a [MonitoredResource][google.api.MonitoredResource] object using a
-// type name and a set of labels. For example, the monitored resource
-// descriptor for Google Compute Engine VM instances has a type of
-// `"gce_instance"` and specifies the use of the labels `"instance_id"` and
-// `"zone"` to identify particular VM instances.
-//
-// Different APIs can support different monitored resource types. APIs generally
-// provide a `list` method that returns the monitored resource descriptors used
-// by the API.
-message MonitoredResourceDescriptor {
- // Optional. The resource name of the monitored resource descriptor:
- // `"projects/{project_id}/monitoredResourceDescriptors/{type}"` where
- // {type} is the value of the `type` field in this object and
- // {project_id} is a project ID that provides API-specific context for
- // accessing the type. APIs that do not use project information can use the
- // resource name format `"monitoredResourceDescriptors/{type}"`.
- string name = 5;
-
- // Required. The monitored resource type. For example, the type
- // `"cloudsql_database"` represents databases in Google Cloud SQL.
- // The maximum length of this value is 256 characters.
- string type = 1;
-
- // Optional. A concise name for the monitored resource type that might be
- // displayed in user interfaces. It should be a Title Cased Noun Phrase,
- // without any article or other determiners. For example,
- // `"Google Cloud SQL Database"`.
- string display_name = 2;
-
- // Optional. A detailed description of the monitored resource type that might
- // be used in documentation.
- string description = 3;
-
- // Required. A set of labels used to describe instances of this monitored
- // resource type. For example, an individual Google Cloud SQL database is
- // identified by values for the labels `"database_id"` and `"zone"`.
- repeated LabelDescriptor labels = 4;
-}
-
-// An object representing a resource that can be used for monitoring, logging,
-// billing, or other purposes. Examples include virtual machine instances,
-// databases, and storage devices such as disks. The `type` field identifies a
-// [MonitoredResourceDescriptor][google.api.MonitoredResourceDescriptor] object that describes the resource's
-// schema. Information in the `labels` field identifies the actual resource and
-// its attributes according to the schema. For example, a particular Compute
-// Engine VM instance could be represented by the following object, because the
-// [MonitoredResourceDescriptor][google.api.MonitoredResourceDescriptor] for `"gce_instance"` has labels
-// `"instance_id"` and `"zone"`:
-//
-// { "type": "gce_instance",
-// "labels": { "instance_id": "12345678901234",
-// "zone": "us-central1-a" }}
-message MonitoredResource {
- // Required. The monitored resource type. This field must match
- // the `type` field of a [MonitoredResourceDescriptor][google.api.MonitoredResourceDescriptor] object. For
- // example, the type of a Compute Engine VM instance is `gce_instance`.
- string type = 1;
-
- // Required. Values for all of the labels listed in the associated monitored
- // resource descriptor. For example, Compute Engine VM instances use the
- // labels `"project_id"`, `"instance_id"`, and `"zone"`.
- map<string, string> labels = 2;
-}
-
-// Auxiliary metadata for a [MonitoredResource][google.api.MonitoredResource] object.
-// [MonitoredResource][google.api.MonitoredResource] objects contain the minimum set of information to
-// uniquely identify a monitored resource instance. There is some other useful
-// auxiliary metadata. Google Stackdriver Monitoring & Logging uses an ingestion
-// pipeline to extract metadata for cloud resources of all types , and stores
-// the metadata in this message.
-message MonitoredResourceMetadata {
- // Output only. Values for predefined system metadata labels.
- // System labels are a kind of metadata extracted by Google Stackdriver.
- // Stackdriver determines what system labels are useful and how to obtain
- // their values. Some examples: "machine_image", "vpc", "subnet_id",
- // "security_group", "name", etc.
- // System label values can be only strings, Boolean values, or a list of
- // strings. For example:
- //
- // { "name": "my-test-instance",
- // "security_group": ["a", "b", "c"],
- // "spot_instance": false }
- google.protobuf.Struct system_labels = 1;
-
- // Output only. A map of user-defined metadata labels.
- map<string, string> user_labels = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/monitoring.proto b/contrib/libs/googleapis-common-protos/google/api/monitoring.proto
deleted file mode 100644
index 30572146dc..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/monitoring.proto
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "MonitoringProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Monitoring configuration of the service.
-//
-// The example below shows how to configure monitored resources and metrics
-// for monitoring. In the example, a monitored resource and two metrics are
-// defined. The `library.googleapis.com/book/returned_count` metric is sent
-// to both producer and consumer projects, whereas the
-// `library.googleapis.com/book/overdue_count` metric is only sent to the
-// consumer project.
-//
-// monitored_resources:
-// - type: library.googleapis.com/branch
-// labels:
-// - key: /city
-// description: The city where the library branch is located in.
-// - key: /name
-// description: The name of the branch.
-// metrics:
-// - name: library.googleapis.com/book/returned_count
-// metric_kind: DELTA
-// value_type: INT64
-// labels:
-// - key: /customer_id
-// - name: library.googleapis.com/book/overdue_count
-// metric_kind: GAUGE
-// value_type: INT64
-// labels:
-// - key: /customer_id
-// monitoring:
-// producer_destinations:
-// - monitored_resource: library.googleapis.com/branch
-// metrics:
-// - library.googleapis.com/book/returned_count
-// consumer_destinations:
-// - monitored_resource: library.googleapis.com/branch
-// metrics:
-// - library.googleapis.com/book/returned_count
-// - library.googleapis.com/book/overdue_count
-message Monitoring {
- // Configuration of a specific monitoring destination (the producer project
- // or the consumer project).
- message MonitoringDestination {
- // The monitored resource type. The type must be defined in
- // [Service.monitored_resources][google.api.Service.monitored_resources] section.
- string monitored_resource = 1;
-
- // Names of the metrics to report to this monitoring destination.
- // Each name must be defined in [Service.metrics][google.api.Service.metrics] section.
- repeated string metrics = 2;
- }
-
- // Monitoring configurations for sending metrics to the producer project.
- // There can be multiple producer destinations, each one must have a
- // different monitored resource type. A metric can be used in at most
- // one producer destination.
- repeated MonitoringDestination producer_destinations = 1;
-
- // Monitoring configurations for sending metrics to the consumer project.
- // There can be multiple consumer destinations, each one must have a
- // different monitored resource type. A metric can be used in at most
- // one consumer destination.
- repeated MonitoringDestination consumer_destinations = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/quota.proto b/contrib/libs/googleapis-common-protos/google/api/quota.proto
deleted file mode 100644
index aa327a1969..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/quota.proto
+++ /dev/null
@@ -1,259 +0,0 @@
-// Copyright 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "QuotaProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Quota configuration helps to achieve fairness and budgeting in service
-// usage.
-//
-// The quota configuration works this way:
-// - The service configuration defines a set of metrics.
-// - For API calls, the quota.metric_rules maps methods to metrics with
-// corresponding costs.
-// - The quota.limits defines limits on the metrics, which will be used for
-// quota checks at runtime.
-//
-// An example quota configuration in yaml format:
-//
-// quota:
-// limits:
-//
-// - name: apiWriteQpsPerProject
-// metric: library.googleapis.com/write_calls
-// unit: "1/min/{project}" # rate limit for consumer projects
-// values:
-// STANDARD: 10000
-//
-// # The metric rules bind all methods to the read_calls metric,
-// # except for the UpdateBook and DeleteBook methods. These two methods
-// # are mapped to the write_calls metric, with the UpdateBook method
-// # consuming at twice rate as the DeleteBook method.
-// metric_rules:
-// - selector: "*"
-// metric_costs:
-// library.googleapis.com/read_calls: 1
-// - selector: google.example.library.v1.LibraryService.UpdateBook
-// metric_costs:
-// library.googleapis.com/write_calls: 2
-// - selector: google.example.library.v1.LibraryService.DeleteBook
-// metric_costs:
-// library.googleapis.com/write_calls: 1
-//
-// Corresponding Metric definition:
-//
-// metrics:
-// - name: library.googleapis.com/read_calls
-// display_name: Read requests
-// metric_kind: DELTA
-// value_type: INT64
-//
-// - name: library.googleapis.com/write_calls
-// display_name: Write requests
-// metric_kind: DELTA
-// value_type: INT64
-//
-message Quota {
- // List of `QuotaLimit` definitions for the service.
- //
- // Used by metric-based quotas only.
- repeated QuotaLimit limits = 3;
-
- // List of `MetricRule` definitions, each one mapping a selected method to one
- // or more metrics.
- //
- // Used by metric-based quotas only.
- repeated MetricRule metric_rules = 4;
-}
-
-// Bind API methods to metrics. Binding a method to a metric causes that
-// metric's configured quota, billing, and monitoring behaviors to apply to the
-// method call.
-//
-// Used by metric-based quotas only.
-message MetricRule {
- // Selects the methods to which this rule applies.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // Metrics to update when the selected methods are called, and the associated
- // cost applied to each metric.
- //
- // The key of the map is the metric name, and the values are the amount
- // increased for the metric against which the quota limits are defined.
- // The value must not be negative.
- map<string, int64> metric_costs = 2;
-}
-
-// `QuotaLimit` defines a specific limit that applies over a specified duration
-// for a limit type. There can be at most one limit for a duration and limit
-// type combination defined within a `QuotaGroup`.
-message QuotaLimit {
- // Name of the quota limit. The name is used to refer to the limit when
- // overriding the default limit on per-consumer basis.
- //
- // For group-based quota limits, the name must be unique within the quota
- // group. If a name is not provided, it will be generated from the limit_by
- // and duration fields.
- //
- // For metric-based quota limits, the name must be provided, and it must be
- // unique within the service. The name can only include alphanumeric
- // characters as well as '-'.
- //
- // The maximum length of the limit name is 64 characters.
- //
- // The name of a limit is used as a unique identifier for this limit.
- // Therefore, once a limit has been put into use, its name should be
- // immutable. You can use the display_name field to provide a user-friendly
- // name for the limit. The display name can be evolved over time without
- // affecting the identity of the limit.
- string name = 6;
-
- // Optional. User-visible, extended description for this quota limit.
- // Should be used only when more context is needed to understand this limit
- // than provided by the limit's display name (see: `display_name`).
- string description = 2;
-
- // Default number of tokens that can be consumed during the specified
- // duration. This is the number of tokens assigned when a client
- // application developer activates the service for his/her project.
- //
- // Specifying a value of 0 will block all requests. This can be used if you
- // are provisioning quota to selected consumers and blocking others.
- // Similarly, a value of -1 will indicate an unlimited quota. No other
- // negative values are allowed.
- //
- // Used by group-based quotas only.
- int64 default_limit = 3;
-
- // Maximum number of tokens that can be consumed during the specified
- // duration. Client application developers can override the default limit up
- // to this maximum. If specified, this value cannot be set to a value less
- // than the default limit. If not specified, it is set to the default limit.
- //
- // To allow clients to apply overrides with no upper bound, set this to -1,
- // indicating unlimited maximum quota.
- //
- // Used by group-based quotas only.
- int64 max_limit = 4;
-
- // Free tier value displayed in the Developers Console for this limit.
- // The free tier is the number of tokens that will be subtracted from the
- // billed amount when billing is enabled.
- // This field can only be set on a limit with duration "1d", in a billable
- // group; it is invalid on any other limit. If this field is not set, it
- // defaults to 0, indicating that there is no free tier for this service.
- //
- // Used by group-based quotas only.
- int64 free_tier = 7;
-
- // Duration of this limit in textual notation. Example: "100s", "24h", "1d".
- // For duration longer than a day, only multiple of days is supported. We
- // support only "100s" and "1d" for now. Additional support will be added in
- // the future. "0" indicates indefinite duration.
- //
- // Used by group-based quotas only.
- string duration = 5;
-
- // The name of the metric this quota limit applies to. The quota limits with
- // the same metric will be checked together during runtime. The metric must be
- // defined within the service config.
- //
- // Used by metric-based quotas only.
- string metric = 8;
-
- // Specify the unit of the quota limit. It uses the same syntax as
- // [Metric.unit][]. The supported unit kinds are determined by the quota
- // backend system.
- //
- // The [Google Service Control](https://cloud.google.com/service-control)
- // supports the following unit components:
- // * One of the time intevals:
- // * "/min" for quota every minute.
- // * "/d" for quota every 24 hours, starting 00:00 US Pacific Time.
- // * Otherwise the quota won't be reset by time, such as storage limit.
- // * One and only one of the granted containers:
- // * "/{organization}" quota for an organization.
- // * "/{project}" quota for a project.
- // * "/{folder}" quota for a folder.
- // * "/{resource}" quota for a universal resource.
- // * Zero or more quota segmentation dimension. Not all combos are valid.
- // * "/{region}" quota for every region. Not to be used with time intervals.
- // * Otherwise the resources granted on the target is not segmented.
- // * "/{zone}" quota for every zone. Not to be used with time intervals.
- // * Otherwise the resources granted on the target is not segmented.
- // * "/{resource}" quota for a resource associated with a project or org.
- //
- // Here are some examples:
- // * "1/min/{project}" for quota per minute per project.
- // * "1/min/{user}" for quota per minute per user.
- // * "1/min/{organization}" for quota per minute per organization.
- //
- // Note: the order of unit components is insignificant.
- // The "1" at the beginning is required to follow the metric unit syntax.
- //
- // Used by metric-based quotas only.
- string unit = 9;
-
- // Tiered limit values. Also allows for regional or zone overrides for these
- // values if "/{region}" or "/{zone}" is specified in the unit field.
- //
- // Currently supported tiers from low to high:
- // VERY_LOW, LOW, STANDARD, HIGH, VERY_HIGH
- //
- // To apply different limit values for users according to their tiers, specify
- // the values for the tiers you want to differentiate. For example:
- // {LOW:100, STANDARD:500, HIGH:1000, VERY_HIGH:5000}
- //
- // The limit value for each tier is optional except for the tier STANDARD.
- // The limit value for an unspecified tier falls to the value of its next
- // tier towards tier STANDARD. For the above example, the limit value for tier
- // STANDARD is 500.
- //
- // To apply the same limit value for all users, just specify limit value for
- // tier STANDARD. For example: {STANDARD:500}.
- //
- // To apply a regional overide for a tier, add a map entry with key
- // "<TIER>/<region>", where <region> is a region name. Similarly, for a zone
- // override, add a map entry with key "<TIER>/{zone}".
- // Further, a wildcard can be used at the end of a zone name in order to
- // specify zone level overrides. For example:
- // LOW: 10, STANDARD: 50, HIGH: 100,
- // LOW/us-central1: 20, STANDARD/us-central1: 60, HIGH/us-central1: 200,
- // LOW/us-central1-*: 10, STANDARD/us-central1-*: 20, HIGH/us-central1-*: 80
- //
- // The regional overrides tier set for each region must be the same as
- // the tier set for default limit values. Same rule applies for zone overrides
- // tier as well.
- //
- // Used by metric-based quotas only.
- map<string, int64> values = 10;
-
- // User-visible display name for this limit.
- // Optional. If not set, the UI will provide a default display name based on
- // the quota configuration. This field can be used to override the default
- // display name generated from the configuration.
- string display_name = 12;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/resource.proto b/contrib/libs/googleapis-common-protos/google/api/resource.proto
deleted file mode 100644
index fd9ee66def..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/resource.proto
+++ /dev/null
@@ -1,299 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/descriptor.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "ResourceProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-extend google.protobuf.FieldOptions {
- // An annotation that describes a resource reference, see
- // [ResourceReference][].
- google.api.ResourceReference resource_reference = 1055;
-}
-
-extend google.protobuf.FileOptions {
- // An annotation that describes a resource definition without a corresponding
- // message; see [ResourceDescriptor][].
- repeated google.api.ResourceDescriptor resource_definition = 1053;
-}
-
-extend google.protobuf.MessageOptions {
- // An annotation that describes a resource definition, see
- // [ResourceDescriptor][].
- google.api.ResourceDescriptor resource = 1053;
-}
-
-// A simple descriptor of a resource type.
-//
-// ResourceDescriptor annotates a resource message (either by means of a
-// protobuf annotation or use in the service config), and associates the
-// resource's schema, the resource type, and the pattern of the resource name.
-//
-// Example:
-//
-// message Topic {
-// // Indicates this message defines a resource schema.
-// // Declares the resource type in the format of {service}/{kind}.
-// // For Kubernetes resources, the format is {api group}/{kind}.
-// option (google.api.resource) = {
-// type: "pubsub.googleapis.com/Topic"
-// name_descriptor: {
-// pattern: "projects/{project}/topics/{topic}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// parent_name_extractor: "projects/{project}"
-// }
-// };
-// }
-//
-// The ResourceDescriptor Yaml config will look like:
-//
-// resources:
-// - type: "pubsub.googleapis.com/Topic"
-// name_descriptor:
-// - pattern: "projects/{project}/topics/{topic}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// parent_name_extractor: "projects/{project}"
-//
-// Sometimes, resources have multiple patterns, typically because they can
-// live under multiple parents.
-//
-// Example:
-//
-// message LogEntry {
-// option (google.api.resource) = {
-// type: "logging.googleapis.com/LogEntry"
-// name_descriptor: {
-// pattern: "projects/{project}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// parent_name_extractor: "projects/{project}"
-// }
-// name_descriptor: {
-// pattern: "folders/{folder}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Folder"
-// parent_name_extractor: "folders/{folder}"
-// }
-// name_descriptor: {
-// pattern: "organizations/{organization}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Organization"
-// parent_name_extractor: "organizations/{organization}"
-// }
-// name_descriptor: {
-// pattern: "billingAccounts/{billing_account}/logs/{log}"
-// parent_type: "billing.googleapis.com/BillingAccount"
-// parent_name_extractor: "billingAccounts/{billing_account}"
-// }
-// };
-// }
-//
-// The ResourceDescriptor Yaml config will look like:
-//
-// resources:
-// - type: 'logging.googleapis.com/LogEntry'
-// name_descriptor:
-// - pattern: "projects/{project}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// parent_name_extractor: "projects/{project}"
-// - pattern: "folders/{folder}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Folder"
-// parent_name_extractor: "folders/{folder}"
-// - pattern: "organizations/{organization}/logs/{log}"
-// parent_type: "cloudresourcemanager.googleapis.com/Organization"
-// parent_name_extractor: "organizations/{organization}"
-// - pattern: "billingAccounts/{billing_account}/logs/{log}"
-// parent_type: "billing.googleapis.com/BillingAccount"
-// parent_name_extractor: "billingAccounts/{billing_account}"
-//
-// For flexible resources, the resource name doesn't contain parent names, but
-// the resource itself has parents for policy evaluation.
-//
-// Example:
-//
-// message Shelf {
-// option (google.api.resource) = {
-// type: "library.googleapis.com/Shelf"
-// name_descriptor: {
-// pattern: "shelves/{shelf}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// }
-// name_descriptor: {
-// pattern: "shelves/{shelf}"
-// parent_type: "cloudresourcemanager.googleapis.com/Folder"
-// }
-// };
-// }
-//
-// The ResourceDescriptor Yaml config will look like:
-//
-// resources:
-// - type: 'library.googleapis.com/Shelf'
-// name_descriptor:
-// - pattern: "shelves/{shelf}"
-// parent_type: "cloudresourcemanager.googleapis.com/Project"
-// - pattern: "shelves/{shelf}"
-// parent_type: "cloudresourcemanager.googleapis.com/Folder"
-message ResourceDescriptor {
- // A description of the historical or future-looking state of the
- // resource pattern.
- enum History {
- // The "unset" value.
- HISTORY_UNSPECIFIED = 0;
-
- // The resource originally had one pattern and launched as such, and
- // additional patterns were added later.
- ORIGINALLY_SINGLE_PATTERN = 1;
-
- // The resource has one pattern, but the API owner expects to add more
- // later. (This is the inverse of ORIGINALLY_SINGLE_PATTERN, and prevents
- // that from being necessary once there are multiple patterns.)
- FUTURE_MULTI_PATTERN = 2;
- }
-
- // A flag representing a specific style that a resource claims to conform to.
- enum Style {
- // The unspecified value. Do not use.
- STYLE_UNSPECIFIED = 0;
-
- // This resource is intended to be "declarative-friendly".
- //
- // Declarative-friendly resources must be more strictly consistent, and
- // setting this to true communicates to tools that this resource should
- // adhere to declarative-friendly expectations.
- //
- // Note: This is used by the API linter (linter.aip.dev) to enable
- // additional checks.
- DECLARATIVE_FRIENDLY = 1;
- }
-
- // The resource type. It must be in the format of
- // {service_name}/{resource_type_kind}. The `resource_type_kind` must be
- // singular and must not include version numbers.
- //
- // Example: `storage.googleapis.com/Bucket`
- //
- // The value of the resource_type_kind must follow the regular expression
- // /[A-Za-z][a-zA-Z0-9]+/. It should start with an upper case character and
- // should use PascalCase (UpperCamelCase). The maximum number of
- // characters allowed for the `resource_type_kind` is 100.
- string type = 1;
-
- // Optional. The relative resource name pattern associated with this resource
- // type. The DNS prefix of the full resource name shouldn't be specified here.
- //
- // The path pattern must follow the syntax, which aligns with HTTP binding
- // syntax:
- //
- // Template = Segment { "/" Segment } ;
- // Segment = LITERAL | Variable ;
- // Variable = "{" LITERAL "}" ;
- //
- // Examples:
- //
- // - "projects/{project}/topics/{topic}"
- // - "projects/{project}/knowledgeBases/{knowledge_base}"
- //
- // The components in braces correspond to the IDs for each resource in the
- // hierarchy. It is expected that, if multiple patterns are provided,
- // the same component name (e.g. "project") refers to IDs of the same
- // type of resource.
- repeated string pattern = 2;
-
- // Optional. The field on the resource that designates the resource name
- // field. If omitted, this is assumed to be "name".
- string name_field = 3;
-
- // Optional. The historical or future-looking state of the resource pattern.
- //
- // Example:
- //
- // // The InspectTemplate message originally only supported resource
- // // names with organization, and project was added later.
- // message InspectTemplate {
- // option (google.api.resource) = {
- // type: "dlp.googleapis.com/InspectTemplate"
- // pattern:
- // "organizations/{organization}/inspectTemplates/{inspect_template}"
- // pattern: "projects/{project}/inspectTemplates/{inspect_template}"
- // history: ORIGINALLY_SINGLE_PATTERN
- // };
- // }
- History history = 4;
-
- // The plural name used in the resource name and permission names, such as
- // 'projects' for the resource name of 'projects/{project}' and the permission
- // name of 'cloudresourcemanager.googleapis.com/projects.get'. It is the same
- // concept of the `plural` field in k8s CRD spec
- // https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
- //
- // Note: The plural form is required even for singleton resources. See
- // https://aip.dev/156
- string plural = 5;
-
- // The same concept of the `singular` field in k8s CRD spec
- // https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
- // Such as "project" for the `resourcemanager.googleapis.com/Project` type.
- string singular = 6;
-
- // Style flag(s) for this resource.
- // These indicate that a resource is expected to conform to a given
- // style. See the specific style flags for additional information.
- repeated Style style = 10;
-}
-
-// Defines a proto annotation that describes a string field that refers to
-// an API resource.
-message ResourceReference {
- // The resource type that the annotated field references.
- //
- // Example:
- //
- // message Subscription {
- // string topic = 2 [(google.api.resource_reference) = {
- // type: "pubsub.googleapis.com/Topic"
- // }];
- // }
- //
- // Occasionally, a field may reference an arbitrary resource. In this case,
- // APIs use the special value * in their resource reference.
- //
- // Example:
- //
- // message GetIamPolicyRequest {
- // string resource = 2 [(google.api.resource_reference) = {
- // type: "*"
- // }];
- // }
- string type = 1;
-
- // The resource type of a child collection that the annotated field
- // references. This is useful for annotating the `parent` field that
- // doesn't have a fixed resource type.
- //
- // Example:
- //
- // message ListLogEntriesRequest {
- // string parent = 1 [(google.api.resource_reference) = {
- // child_type: "logging.googleapis.com/LogEntry"
- // };
- // }
- string child_type = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/routing.proto b/contrib/libs/googleapis-common-protos/google/api/routing.proto
deleted file mode 100644
index 0138283890..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/routing.proto
+++ /dev/null
@@ -1,461 +0,0 @@
-// Copyright 2021 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
-option java_multiple_files = true;
-option java_outer_classname = "RoutingProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-extend google.protobuf.MethodOptions {
- // See RoutingRule.
- google.api.RoutingRule routing = 72295729;
-}
-
-// Specifies the routing information that should be sent along with the request
-// in the form of routing header.
-// **NOTE:** All service configuration rules follow the "last one wins" order.
-//
-// The examples below will apply to an RPC which has the following request type:
-//
-// Message Definition:
-//
-// message Request {
-// // The name of the Table
-// // Values can be of the following formats:
-// // - `projects/<project>/tables/<table>`
-// // - `projects/<project>/instances/<instance>/tables/<table>`
-// // - `region/<region>/zones/<zone>/tables/<table>`
-// string table_name = 1;
-//
-// // This value specifies routing for replication.
-// // It can be in the following formats:
-// // - `profiles/<profile_id>`
-// // - a legacy `profile_id` that can be any string
-// string app_profile_id = 2;
-// }
-//
-// Example message:
-//
-// {
-// table_name: projects/proj_foo/instances/instance_bar/table/table_baz,
-// app_profile_id: profiles/prof_qux
-// }
-//
-// The routing header consists of one or multiple key-value pairs. Every key
-// and value must be percent-encoded, and joined together in the format of
-// `key1=value1&key2=value2`.
-// In the examples below I am skipping the percent-encoding for readablity.
-//
-// Example 1
-//
-// Extracting a field from the request to put into the routing header
-// unchanged, with the key equal to the field name.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take the `app_profile_id`.
-// routing_parameters {
-// field: "app_profile_id"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params: app_profile_id=profiles/prof_qux
-//
-// Example 2
-//
-// Extracting a field from the request to put into the routing header
-// unchanged, with the key different from the field name.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take the `app_profile_id`, but name it `routing_id` in the header.
-// routing_parameters {
-// field: "app_profile_id"
-// path_template: "{routing_id=**}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params: routing_id=profiles/prof_qux
-//
-// Example 3
-//
-// Extracting a field from the request to put into the routing
-// header, while matching a path template syntax on the field's value.
-//
-// NB: it is more useful to send nothing than to send garbage for the purpose
-// of dynamic routing, since garbage pollutes cache. Thus the matching.
-//
-// Sub-example 3a
-//
-// The field matches the template.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take the `table_name`, if it's well-formed (with project-based
-// // syntax).
-// routing_parameters {
-// field: "table_name"
-// path_template: "{table_name=projects/*/instances/*/**}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// table_name=projects/proj_foo/instances/instance_bar/table/table_baz
-//
-// Sub-example 3b
-//
-// The field does not match the template.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take the `table_name`, if it's well-formed (with region-based
-// // syntax).
-// routing_parameters {
-// field: "table_name"
-// path_template: "{table_name=regions/*/zones/*/**}"
-// }
-// };
-//
-// result:
-//
-// <no routing header will be sent>
-//
-// Sub-example 3c
-//
-// Multiple alternative conflictingly named path templates are
-// specified. The one that matches is used to construct the header.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take the `table_name`, if it's well-formed, whether
-// // using the region- or projects-based syntax.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{table_name=regions/*/zones/*/**}"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "{table_name=projects/*/instances/*/**}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// table_name=projects/proj_foo/instances/instance_bar/table/table_baz
-//
-// Example 4
-//
-// Extracting a single routing header key-value pair by matching a
-// template syntax on (a part of) a single request field.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // Take just the project id from the `table_name` field.
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=projects/*}/**"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params: routing_id=projects/proj_foo
-//
-// Example 5
-//
-// Extracting a single routing header key-value pair by matching
-// several conflictingly named path templates on (parts of) a single request
-// field. The last template to match "wins" the conflict.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // If the `table_name` does not have instances information,
-// // take just the project id for routing.
-// // Otherwise take project + instance.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=projects/*}/**"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=projects/*/instances/*}/**"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// routing_id=projects/proj_foo/instances/instance_bar
-//
-// Example 6
-//
-// Extracting multiple routing header key-value pairs by matching
-// several non-conflicting path templates on (parts of) a single request field.
-//
-// Sub-example 6a
-//
-// Make the templates strict, so that if the `table_name` does not
-// have an instance information, nothing is sent.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // The routing code needs two keys instead of one composite
-// // but works only for the tables with the "project-instance" name
-// // syntax.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{project_id=projects/*}/instances/*/**"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "projects/*/{instance_id=instances/*}/**"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// project_id=projects/proj_foo&instance_id=instances/instance_bar
-//
-// Sub-example 6b
-//
-// Make the templates loose, so that if the `table_name` does not
-// have an instance information, just the project id part is sent.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // The routing code wants two keys instead of one composite
-// // but will work with just the `project_id` for tables without
-// // an instance in the `table_name`.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{project_id=projects/*}/**"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "projects/*/{instance_id=instances/*}/**"
-// }
-// };
-//
-// result (is the same as 6a for our example message because it has the instance
-// information):
-//
-// x-goog-request-params:
-// project_id=projects/proj_foo&instance_id=instances/instance_bar
-//
-// Example 7
-//
-// Extracting multiple routing header key-value pairs by matching
-// several path templates on multiple request fields.
-//
-// NB: note that here there is no way to specify sending nothing if one of the
-// fields does not match its template. E.g. if the `table_name` is in the wrong
-// format, the `project_id` will not be sent, but the `routing_id` will be.
-// The backend routing code has to be aware of that and be prepared to not
-// receive a full complement of keys if it expects multiple.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // The routing needs both `project_id` and `routing_id`
-// // (from the `app_profile_id` field) for routing.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{project_id=projects/*}/**"
-// }
-// routing_parameters {
-// field: "app_profile_id"
-// path_template: "{routing_id=**}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// project_id=projects/proj_foo&routing_id=profiles/prof_qux
-//
-// Example 8
-//
-// Extracting a single routing header key-value pair by matching
-// several conflictingly named path templates on several request fields. The
-// last template to match "wins" the conflict.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // The `routing_id` can be a project id or a region id depending on
-// // the table name format, but only if the `app_profile_id` is not set.
-// // If `app_profile_id` is set it should be used instead.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=projects/*}/**"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=regions/*}/**"
-// }
-// routing_parameters {
-// field: "app_profile_id"
-// path_template: "{routing_id=**}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params: routing_id=profiles/prof_qux
-//
-// Example 9
-//
-// Bringing it all together.
-//
-// annotation:
-//
-// option (google.api.routing) = {
-// // For routing both `table_location` and a `routing_id` are needed.
-// //
-// // table_location can be either an instance id or a region+zone id.
-// //
-// // For `routing_id`, take the value of `app_profile_id`
-// // - If it's in the format `profiles/<profile_id>`, send
-// // just the `<profile_id>` part.
-// // - If it's any other literal, send it as is.
-// // If the `app_profile_id` is empty, and the `table_name` starts with
-// // the project_id, send that instead.
-//
-// routing_parameters {
-// field: "table_name"
-// path_template: "projects/*/{table_location=instances/*}/tables/*"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "{table_location=regions/*/zones/*}/tables/*"
-// }
-// routing_parameters {
-// field: "table_name"
-// path_template: "{routing_id=projects/*}/**"
-// }
-// routing_parameters {
-// field: "app_profile_id"
-// path_template: "{routing_id=**}"
-// }
-// routing_parameters {
-// field: "app_profile_id"
-// path_template: "profiles/{routing_id=*}"
-// }
-// };
-//
-// result:
-//
-// x-goog-request-params:
-// table_location=instances/instance_bar&routing_id=prof_qux
-message RoutingRule {
- // A collection of Routing Parameter specifications.
- // **NOTE:** If multiple Routing Parameters describe the same key
- // (via the `path_template` field or via the `field` field when
- // `path_template` is not provided), "last one wins" rule
- // determines which Parameter gets used.
- // See the examples for more details.
- repeated RoutingParameter routing_parameters = 2;
-}
-
-// A projection from an input message to the GRPC or REST header.
-message RoutingParameter {
- // A request field to extract the header key-value pair from.
- string field = 1;
-
- // A pattern matching the key-value field. Optional.
- // If not specified, the whole field specified in the `field` field will be
- // taken as value, and its name used as key. If specified, it MUST contain
- // exactly one named segment (along with any number of unnamed segments) The
- // pattern will be matched over the field specified in the `field` field, then
- // if the match is successful:
- // - the name of the single named segment will be used as a header name,
- // - the match value of the segment will be used as a header value;
- // if the match is NOT successful, nothing will be sent.
- //
- // Example:
- //
- // -- This is a field in the request message
- // | that the header value will be extracted from.
- // |
- // | -- This is the key name in the
- // | | routing header.
- // V |
- // field: "table_name" v
- // path_template: "projects/*/{table_location=instances/*}/tables/*"
- // ^ ^
- // | |
- // In the {} brackets is the pattern that -- |
- // specifies what to extract from the |
- // field as a value to be sent. |
- // |
- // The string in the field must match the whole pattern --
- // before brackets, inside brackets, after brackets.
- //
- // When looking at this specific example, we can see that:
- // - A key-value pair with the key `table_location`
- // and the value matching `instances/*` should be added
- // to the x-goog-request-params routing header.
- // - The value is extracted from the request message's `table_name` field
- // if it matches the full pattern specified:
- // `projects/*/instances/*/tables/*`.
- //
- // **NB:** If the `path_template` field is not provided, the key name is
- // equal to the field name, and the whole field should be sent as a value.
- // This makes the pattern for the field and the value functionally equivalent
- // to `**`, and the configuration
- //
- // {
- // field: "table_name"
- // }
- //
- // is a functionally equivalent shorthand to:
- //
- // {
- // field: "table_name"
- // path_template: "{table_name=**}"
- // }
- //
- // See Example 1 for more details.
- string path_template = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/service.proto b/contrib/libs/googleapis-common-protos/google/api/service.proto
deleted file mode 100644
index 951c7ac050..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/service.proto
+++ /dev/null
@@ -1,175 +0,0 @@
-// Copyright 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-import "google/api/auth.proto";
-import "google/api/backend.proto";
-import "google/api/billing.proto";
-import "google/api/context.proto";
-import "google/api/control.proto";
-import "google/api/documentation.proto";
-import "google/api/endpoint.proto";
-import "google/api/http.proto";
-import "google/api/log.proto";
-import "google/api/logging.proto";
-import "google/api/metric.proto";
-import "google/api/monitored_resource.proto";
-import "google/api/monitoring.proto";
-import "google/api/quota.proto";
-import "google/api/source_info.proto";
-import "google/api/system_parameter.proto";
-import "google/api/usage.proto";
-import "google/protobuf/api.proto";
-import "google/protobuf/type.proto";
-import "google/protobuf/wrappers.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "ServiceProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// `Service` is the root object of Google service configuration schema. It
-// describes basic information about a service, such as the name and the
-// title, and delegates other aspects to sub-sections. Each sub-section is
-// either a proto message or a repeated proto message that configures a
-// specific aspect, such as auth. See each proto message definition for details.
-//
-// Example:
-//
-// type: google.api.Service
-// config_version: 3
-// name: calendar.googleapis.com
-// title: Google Calendar API
-// apis:
-// - name: google.calendar.v3.Calendar
-// authentication:
-// providers:
-// - id: google_calendar_auth
-// jwks_uri: https://www.googleapis.com/oauth2/v1/certs
-// issuer: https://securetoken.google.com
-// rules:
-// - selector: "*"
-// requirements:
-// provider_id: google_calendar_auth
-message Service {
- // The semantic version of the service configuration. The config version
- // affects the interpretation of the service configuration. For example,
- // certain features are enabled by default for certain config versions.
- // The latest config version is `3`.
- google.protobuf.UInt32Value config_version = 20;
-
- // The DNS address at which this service is available,
- // e.g. `calendar.googleapis.com`.
- string name = 1;
-
- // A unique ID for a specific instance of this message, typically assigned
- // by the client for tracking purpose. If empty, the server may choose to
- // generate one instead.
- string id = 33;
-
- // The product title for this service.
- string title = 2;
-
- // The Google project that owns this service.
- string producer_project_id = 22;
-
- // A list of API interfaces exported by this service. Only the `name` field
- // of the [google.protobuf.Api][google.protobuf.Api] needs to be provided by the configuration
- // author, as the remaining fields will be derived from the IDL during the
- // normalization process. It is an error to specify an API interface here
- // which cannot be resolved against the associated IDL files.
- repeated google.protobuf.Api apis = 3;
-
- // A list of all proto message types included in this API service.
- // Types referenced directly or indirectly by the `apis` are
- // automatically included. Messages which are not referenced but
- // shall be included, such as types used by the `google.protobuf.Any` type,
- // should be listed here by name. Example:
- //
- // types:
- // - name: google.protobuf.Int32
- repeated google.protobuf.Type types = 4;
-
- // A list of all enum types included in this API service. Enums
- // referenced directly or indirectly by the `apis` are automatically
- // included. Enums which are not referenced but shall be included
- // should be listed here by name. Example:
- //
- // enums:
- // - name: google.someapi.v1.SomeEnum
- repeated google.protobuf.Enum enums = 5;
-
- // Additional API documentation.
- Documentation documentation = 6;
-
- // API backend configuration.
- Backend backend = 8;
-
- // HTTP configuration.
- Http http = 9;
-
- // Quota configuration.
- Quota quota = 10;
-
- // Auth configuration.
- Authentication authentication = 11;
-
- // Context configuration.
- Context context = 12;
-
- // Configuration controlling usage of this service.
- Usage usage = 15;
-
- // Configuration for network endpoints. If this is empty, then an endpoint
- // with the same name as the service is automatically generated to service all
- // defined APIs.
- repeated Endpoint endpoints = 18;
-
- // Configuration for the service control plane.
- Control control = 21;
-
- // Defines the logs used by this service.
- repeated LogDescriptor logs = 23;
-
- // Defines the metrics used by this service.
- repeated MetricDescriptor metrics = 24;
-
- // Defines the monitored resources used by this service. This is required
- // by the [Service.monitoring][google.api.Service.monitoring] and [Service.logging][google.api.Service.logging] configurations.
- repeated MonitoredResourceDescriptor monitored_resources = 25;
-
- // Billing configuration.
- Billing billing = 26;
-
- // Logging configuration.
- Logging logging = 27;
-
- // Monitoring configuration.
- Monitoring monitoring = 28;
-
- // System parameter configuration.
- SystemParameters system_parameters = 29;
-
- // Output only. The source information for this configuration if available.
- SourceInfo source_info = 37;
-
- // Experimental configuration.
- reserved 101;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/source_info.proto b/contrib/libs/googleapis-common-protos/google/api/source_info.proto
deleted file mode 100644
index 58777856fb..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/source_info.proto
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/protobuf/any.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "SourceInfoProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Source information used to create a Service Config
-message SourceInfo {
- // All files used during config generation.
- repeated google.protobuf.Any source_files = 1;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/system_parameter.proto b/contrib/libs/googleapis-common-protos/google/api/system_parameter.proto
deleted file mode 100644
index a55212f044..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/system_parameter.proto
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "SystemParameterProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// ### System parameter configuration
-//
-// A system parameter is a special kind of parameter defined by the API
-// system, not by an individual API. It is typically mapped to an HTTP header
-// and/or a URL query parameter. This configuration specifies which methods
-// change the names of the system parameters.
-message SystemParameters {
- // Define system parameters.
- //
- // The parameters defined here will override the default parameters
- // implemented by the system. If this field is missing from the service
- // config, default system parameters will be used. Default system parameters
- // and names is implementation-dependent.
- //
- // Example: define api key for all methods
- //
- // system_parameters
- // rules:
- // - selector: "*"
- // parameters:
- // - name: api_key
- // url_query_parameter: api_key
- //
- //
- // Example: define 2 api key names for a specific method.
- //
- // system_parameters
- // rules:
- // - selector: "/ListShelves"
- // parameters:
- // - name: api_key
- // http_header: Api-Key1
- // - name: api_key
- // http_header: Api-Key2
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated SystemParameterRule rules = 1;
-}
-
-// Define a system parameter rule mapping system parameter definitions to
-// methods.
-message SystemParameterRule {
- // Selects the methods to which this rule applies. Use '*' to indicate all
- // methods in all APIs.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // Define parameters. Multiple names may be defined for a parameter.
- // For a given method call, only one of them should be used. If multiple
- // names are used the behavior is implementation-dependent.
- // If none of the specified names are present the behavior is
- // parameter-dependent.
- repeated SystemParameter parameters = 2;
-}
-
-// Define a parameter's name and location. The parameter may be passed as either
-// an HTTP header or a URL query parameter, and if both are passed the behavior
-// is implementation-dependent.
-message SystemParameter {
- // Define the name of the parameter, such as "api_key" . It is case sensitive.
- string name = 1;
-
- // Define the HTTP header name to use for the parameter. It is case
- // insensitive.
- string http_header = 2;
-
- // Define the URL query parameter name to use for the parameter. It is case
- // sensitive.
- string url_query_parameter = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/api/usage.proto b/contrib/libs/googleapis-common-protos/google/api/usage.proto
deleted file mode 100644
index 63299b56d2..0000000000
--- a/contrib/libs/googleapis-common-protos/google/api/usage.proto
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2018 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.api;
-
-import "google/api/annotations.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/api/serviceconfig;serviceconfig";
-option java_multiple_files = true;
-option java_outer_classname = "UsageProto";
-option java_package = "com.google.api";
-option objc_class_prefix = "GAPI";
-
-
-// Configuration controlling usage of a service.
-message Usage {
- // Requirements that must be satisfied before a consumer project can use the
- // service. Each requirement is of the form <service.name>/<requirement-id>;
- // for example 'serviceusage.googleapis.com/billing-enabled'.
- repeated string requirements = 1;
-
- // A list of usage rules that apply to individual API methods.
- //
- // **NOTE:** All service configuration rules follow "last one wins" order.
- repeated UsageRule rules = 6;
-
- // The full resource name of a channel used for sending notifications to the
- // service producer.
- //
- // Google Service Management currently only supports
- // [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) as a notification
- // channel. To use Google Cloud Pub/Sub as the channel, this must be the name
- // of a Cloud Pub/Sub topic that uses the Cloud Pub/Sub topic name format
- // documented in https://cloud.google.com/pubsub/docs/overview.
- string producer_notification_channel = 7;
-}
-
-// Usage configuration rules for the service.
-//
-// NOTE: Under development.
-//
-//
-// Use this rule to configure unregistered calls for the service. Unregistered
-// calls are calls that do not contain consumer project identity.
-// (Example: calls that do not contain an API key).
-// By default, API methods do not allow unregistered calls, and each method call
-// must be identified by a consumer project identity. Use this rule to
-// allow/disallow unregistered calls.
-//
-// Example of an API that wants to allow unregistered calls for entire service.
-//
-// usage:
-// rules:
-// - selector: "*"
-// allow_unregistered_calls: true
-//
-// Example of a method that wants to allow unregistered calls.
-//
-// usage:
-// rules:
-// - selector: "google.example.library.v1.LibraryService.CreateBook"
-// allow_unregistered_calls: true
-message UsageRule {
- // Selects the methods to which this rule applies. Use '*' to indicate all
- // methods in all APIs.
- //
- // Refer to [selector][google.api.DocumentationRule.selector] for syntax details.
- string selector = 1;
-
- // If true, the selected method allows unregistered calls, e.g. calls
- // that don't identify any user or application.
- bool allow_unregistered_calls = 2;
-
- // If true, the selected method should skip service control and the control
- // plane features, such as quota and billing, will not be available.
- // This flag is used by Google Cloud Endpoints to bypass checks for internal
- // methods, such as service health check methods.
- bool skip_service_control = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/cloud/extended_operations.proto b/contrib/libs/googleapis-common-protos/google/cloud/extended_operations.proto
deleted file mode 100644
index 1477d2d69a..0000000000
--- a/contrib/libs/googleapis-common-protos/google/cloud/extended_operations.proto
+++ /dev/null
@@ -1,150 +0,0 @@
-// Copyright 2021 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// This file contains custom annotations that are used by GAPIC generators to
-// handle Long Running Operation methods (LRO) that are NOT compliant with
-// https://google.aip.dev/151. These annotations are public for technical
-// reasons only. Please DO NOT USE them in your protos.
-syntax = "proto3";
-
-package google.cloud;
-
-import "google/protobuf/descriptor.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/cloud/extendedops;extendedops";
-option java_multiple_files = true;
-option java_outer_classname = "ExtendedOperationsProto";
-option java_package = "com.google.cloud";
-option objc_class_prefix = "GAPI";
-
-// FieldOptions to match corresponding fields in the initial request,
-// polling request and operation response messages.
-//
-// Example:
-//
-// In an API-specific operation message:
-//
-// message MyOperation {
-// string http_error_message = 1 [(operation_field) = ERROR_MESSAGE];
-// int32 http_error_status_code = 2 [(operation_field) = ERROR_CODE];
-// string id = 3 [(operation_field) = NAME];
-// Status status = 4 [(operation_field) = STATUS];
-// }
-//
-// In a polling request message (the one which is used to poll for an LRO
-// status):
-//
-// message MyPollingRequest {
-// string operation = 1 [(operation_response_field) = "id"];
-// string project = 2;
-// string region = 3;
-// }
-//
-// In an initial request message (the one which starts an LRO):
-//
-// message MyInitialRequest {
-// string my_project = 2 [(operation_request_field) = "project"];
-// string my_region = 3 [(operation_request_field) = "region"];
-// }
-//
-extend google.protobuf.FieldOptions {
- // A field annotation that maps fields in an API-specific Operation object to
- // their standard counterparts in google.longrunning.Operation. See
- // OperationResponseMapping enum definition.
- OperationResponseMapping operation_field = 1149;
-
- // A field annotation that maps fields in the initial request message
- // (the one which started the LRO) to their counterparts in the polling
- // request message. For non-standard LRO, the polling response may be missing
- // some of the information needed to make a subsequent polling request. The
- // missing information (for example, project or region ID) is contained in the
- // fields of the initial request message that this annotation must be applied
- // to. The string value of the annotation corresponds to the name of the
- // counterpart field in the polling request message that the annotated field's
- // value will be copied to.
- string operation_request_field = 1150;
-
- // A field annotation that maps fields in the polling request message to their
- // counterparts in the initial and/or polling response message. The initial
- // and the polling methods return an API-specific Operation object. Some of
- // the fields from that response object must be reused in the subsequent
- // request (like operation name/ID) to fully identify the polled operation.
- // This annotation must be applied to the fields in the polling request
- // message, the string value of the annotation must correspond to the name of
- // the counterpart field in the Operation response object whose value will be
- // copied to the annotated field.
- string operation_response_field = 1151;
-}
-
-// MethodOptions to identify the actual service and method used for operation
-// status polling.
-//
-// Example:
-//
-// In a method, which starts an LRO:
-//
-// service MyService {
-// rpc Foo(MyInitialRequest) returns (MyOperation) {
-// option (operation_service) = "MyPollingService";
-// }
-// }
-//
-// In a polling method:
-//
-// service MyPollingService {
-// rpc Get(MyPollingRequest) returns (MyOperation) {
-// option (operation_polling_method) = true;
-// }
-// }
-extend google.protobuf.MethodOptions {
- // A method annotation that maps an LRO method (the one which starts an LRO)
- // to the service, which will be used to poll for the operation status. The
- // annotation must be applied to the method which starts an LRO, the string
- // value of the annotation must correspond to the name of the service used to
- // poll for the operation status.
- string operation_service = 1249;
-
- // A method annotation that marks methods that can be used for polling
- // operation status (e.g. the MyPollingService.Get(MyPollingRequest) method).
- bool operation_polling_method = 1250;
-}
-
-// An enum to be used to mark the essential (for polling) fields in an
-// API-specific Operation object. A custom Operation object may contain many
-// different fields, but only few of them are essential to conduct a successful
-// polling process.
-enum OperationResponseMapping {
- // Do not use.
- UNDEFINED = 0;
-
- // A field in an API-specific (custom) Operation object which carries the same
- // meaning as google.longrunning.Operation.name.
- NAME = 1;
-
- // A field in an API-specific (custom) Operation object which carries the same
- // meaning as google.longrunning.Operation.done. If the annotated field is of
- // an enum type, `annotated_field_name == EnumType.DONE` semantics should be
- // equivalent to `Operation.done == true`. If the annotated field is of type
- // boolean, then it should follow the same semantics as Operation.done.
- // Otherwise, a non-empty value should be treated as `Operation.done == true`.
- STATUS = 2;
-
- // A field in an API-specific (custom) Operation object which carries the same
- // meaning as google.longrunning.Operation.error.code.
- ERROR_CODE = 3;
-
- // A field in an API-specific (custom) Operation object which carries the same
- // meaning as google.longrunning.Operation.error.message.
- ERROR_MESSAGE = 4;
-} \ No newline at end of file
diff --git a/contrib/libs/googleapis-common-protos/google/cloud/location/locations.proto b/contrib/libs/googleapis-common-protos/google/cloud/location/locations.proto
deleted file mode 100644
index b210cc3230..0000000000
--- a/contrib/libs/googleapis-common-protos/google/cloud/location/locations.proto
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.cloud.location;
-
-import "google/api/annotations.proto";
-import "google/protobuf/any.proto";
-import "google/api/client.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/cloud/location;location";
-option java_multiple_files = true;
-option java_outer_classname = "LocationsProto";
-option java_package = "com.google.cloud.location";
-
-// An abstract interface that provides location-related information for
-// a service. Service-specific metadata is provided through the
-// [Location.metadata][google.cloud.location.Location.metadata] field.
-service Locations {
- option (google.api.default_host) = "cloud.googleapis.com";
- option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
-
- // Lists information about the supported locations for this service.
- rpc ListLocations(ListLocationsRequest) returns (ListLocationsResponse) {
- option (google.api.http) = {
- get: "/v1/{name=locations}"
- additional_bindings {
- get: "/v1/{name=projects/*}/locations"
- }
- };
- }
-
- // Gets information about a location.
- rpc GetLocation(GetLocationRequest) returns (Location) {
- option (google.api.http) = {
- get: "/v1/{name=locations/*}"
- additional_bindings {
- get: "/v1/{name=projects/*/locations/*}"
- }
- };
- }
-}
-
-// The request message for [Locations.ListLocations][google.cloud.location.Locations.ListLocations].
-message ListLocationsRequest {
- // The resource that owns the locations collection, if applicable.
- string name = 1;
-
- // The standard list filter.
- string filter = 2;
-
- // The standard list page size.
- int32 page_size = 3;
-
- // The standard list page token.
- string page_token = 4;
-}
-
-// The response message for [Locations.ListLocations][google.cloud.location.Locations.ListLocations].
-message ListLocationsResponse {
- // A list of locations that matches the specified filter in the request.
- repeated Location locations = 1;
-
- // The standard List next-page token.
- string next_page_token = 2;
-}
-
-// The request message for [Locations.GetLocation][google.cloud.location.Locations.GetLocation].
-message GetLocationRequest {
- // Resource name for the location.
- string name = 1;
-}
-
-// A resource that represents Google Cloud Platform location.
-message Location {
- // Resource name for the location, which may vary between implementations.
- // For example: `"projects/example-project/locations/us-east1"`
- string name = 1;
-
- // The canonical id for this location. For example: `"us-east1"`.
- string location_id = 4;
-
- // The friendly name for this location, typically a nearby city name.
- // For example, "Tokyo".
- string display_name = 5;
-
- // Cross-service attributes for the location. For example
- //
- // {"cloud.googleapis.com/region": "us-east1"}
- map<string, string> labels = 2;
-
- // Service-specific metadata. For example the available capacity at the given
- // location.
- google.protobuf.Any metadata = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/gapic/metadata/gapic_metadata.proto b/contrib/libs/googleapis-common-protos/google/gapic/metadata/gapic_metadata.proto
deleted file mode 100644
index 16090ede07..0000000000
--- a/contrib/libs/googleapis-common-protos/google/gapic/metadata/gapic_metadata.proto
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2020 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.gapic.metadata;
-
-option csharp_namespace = "Google.Gapic.Metadata";
-option go_package = "google.golang.org/genproto/googleapis/gapic/metadata;metadata";
-option java_multiple_files = true;
-option java_outer_classname = "GapicMetadataProto";
-option java_package = "com.google.gapic.metadata";
-option php_namespace = "Google\\Gapic\\Metadata";
-option ruby_package = "Google::Gapic::Metadata";
-
-// Metadata about a GAPIC library for a specific combination of API, version, and
-// computer language.
-message GapicMetadata {
- // Schema version of this proto. Current value: 1.0
- string schema = 1;
-
- // Any human-readable comments to be included in this file.
- string comment = 2;
-
- // Computer language of this generated language. This must be
- // spelled out as it spoken in English, with no capitalization or
- // separators (e.g. "csharp", "nodejs").
- string language = 3;
-
- // The proto package containing the API definition for which this
- // GAPIC library was generated.
- string proto_package = 4;
-
- // The language-specific library package for this GAPIC library.
- string library_package = 5;
-
- // A map from each proto-defined service to ServiceForTransports,
- // which allows listing information about transport-specific
- // implementations of the service.
- //
- // The key is the name of the service as it appears in the .proto
- // file.
- map<string, ServiceForTransport> services = 6;
-
- // A map from a transport name to ServiceAsClient, which allows
- // listing information about the client objects that implement the
- // parent RPC service for the specified transport.
- //
- // The key name is the transport, lower-cased with no separators
- // (e.g. "grpc", "rest").
- message ServiceForTransport {
- map<string, ServiceAsClient> clients = 1;
- }
-
- // Information about a specific client implementing a proto-defined service.
- message ServiceAsClient {
- // The name of the library client formatted as it appears in the source code
- string library_client = 1;
-
- // A mapping from each proto-defined RPC name to the the list of
- // methods in library_client that implement it. There can be more
- // than one library_client method for each RPC. RPCs with no
- // library_client methods need not be included.
- //
- // The key name is the name of the RPC as defined and formated in
- // the proto file.
- map<string, MethodList> rpcs = 2;
- }
-
- // List of GAPIC client methods implementing the proto-defined RPC
- // for the transport and service specified in the containing
- // structures.
- message MethodList {
- // List of methods for a specific proto-service client in the
- // GAPIC. These names should be formatted as they appear in the
- // source code.
- repeated string methods = 1;
- }
-
-}
diff --git a/contrib/libs/googleapis-common-protos/google/logging/type/README.md b/contrib/libs/googleapis-common-protos/google/logging/type/README.md
deleted file mode 100644
index 2d51dc0417..0000000000
--- a/contrib/libs/googleapis-common-protos/google/logging/type/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-## Logging types
-
-This package contains shared [protocol buffer][protobuf] types that are populated
-by the Stackdriver Logging API and consumed by other APIs.
-
-### Key Concepts
-
-- **HttpRequest**: Contains the complete set of information about a particular
- HTTP request, such as HTTP method, request URL, status code, and other things.
-- **LogSeverity**: The severity of a log entry (e.g. `DEBUG`, `INFO`, `WARNING`).
-
-[protobuf]: https://developers.google.com/protocol-buffers/
diff --git a/contrib/libs/googleapis-common-protos/google/logging/type/http_request.proto b/contrib/libs/googleapis-common-protos/google/logging/type/http_request.proto
deleted file mode 100644
index 8b9301f117..0000000000
--- a/contrib/libs/googleapis-common-protos/google/logging/type/http_request.proto
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.logging.type;
-
-import "google/api/annotations.proto";
-import "google/protobuf/duration.proto";
-
-option csharp_namespace = "Google.Cloud.Logging.Type";
-option go_package = "google.golang.org/genproto/googleapis/logging/type;ltype";
-option java_multiple_files = true;
-option java_outer_classname = "HttpRequestProto";
-option java_package = "com.google.logging.type";
-option php_namespace = "Google\\Cloud\\Logging\\Type";
-
-
-// A common proto for logging HTTP requests. Only contains semantics
-// defined by the HTTP specification. Product-specific logging
-// information MUST be defined in a separate message.
-message HttpRequest {
- // The request method. Examples: `"GET"`, `"HEAD"`, `"PUT"`, `"POST"`.
- string request_method = 1;
-
- // The scheme (http, https), the host name, the path and the query
- // portion of the URL that was requested.
- // Example: `"http://example.com/some/info?color=red"`.
- string request_url = 2;
-
- // The size of the HTTP request message in bytes, including the request
- // headers and the request body.
- int64 request_size = 3;
-
- // The response code indicating the status of response.
- // Examples: 200, 404.
- int32 status = 4;
-
- // The size of the HTTP response message sent back to the client, in bytes,
- // including the response headers and the response body.
- int64 response_size = 5;
-
- // The user agent sent by the client. Example:
- // `"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Q312461; .NET CLR 1.0.3705)"`.
- string user_agent = 6;
-
- // The IP address (IPv4 or IPv6) of the client that issued the HTTP
- // request. Examples: `"192.168.1.1"`, `"FE80::0202:B3FF:FE1E:8329"`.
- string remote_ip = 7;
-
- // The IP address (IPv4 or IPv6) of the origin server that the request was
- // sent to.
- string server_ip = 13;
-
- // The referer URL of the request, as defined in
- // [HTTP/1.1 Header Field Definitions](http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html).
- string referer = 8;
-
- // The request processing latency on the server, from the time the request was
- // received until the response was sent.
- google.protobuf.Duration latency = 14;
-
- // Whether or not a cache lookup was attempted.
- bool cache_lookup = 11;
-
- // Whether or not an entity was served from cache
- // (with or without validation).
- bool cache_hit = 9;
-
- // Whether or not the response was validated with the origin server before
- // being served from cache. This field is only meaningful if `cache_hit` is
- // True.
- bool cache_validated_with_origin_server = 10;
-
- // The number of HTTP response bytes inserted into cache. Set only when a
- // cache fill was attempted.
- int64 cache_fill_bytes = 12;
-
- // Protocol used for the request. Examples: "HTTP/1.1", "HTTP/2", "websocket"
- string protocol = 15;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/logging/type/log_severity.proto b/contrib/libs/googleapis-common-protos/google/logging/type/log_severity.proto
deleted file mode 100644
index 41351c6765..0000000000
--- a/contrib/libs/googleapis-common-protos/google/logging/type/log_severity.proto
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.logging.type;
-
-import "google/api/annotations.proto";
-
-option csharp_namespace = "Google.Cloud.Logging.Type";
-option go_package = "google.golang.org/genproto/googleapis/logging/type;ltype";
-option java_multiple_files = true;
-option java_outer_classname = "LogSeverityProto";
-option java_package = "com.google.logging.type";
-option php_namespace = "Google\\Cloud\\Logging\\Type";
-
-
-// The severity of the event described in a log entry, expressed as one of the
-// standard severity levels listed below. For your reference, the levels are
-// assigned the listed numeric values. The effect of using numeric values other
-// than those listed is undefined.
-//
-// You can filter for log entries by severity. For example, the following
-// filter expression will match log entries with severities `INFO`, `NOTICE`,
-// and `WARNING`:
-//
-// severity > DEBUG AND severity <= WARNING
-//
-// If you are writing log entries, you should map other severity encodings to
-// one of these standard levels. For example, you might map all of Java's FINE,
-// FINER, and FINEST levels to `LogSeverity.DEBUG`. You can preserve the
-// original severity level in the log entry payload if you wish.
-enum LogSeverity {
- // (0) The log entry has no assigned severity level.
- DEFAULT = 0;
-
- // (100) Debug or trace information.
- DEBUG = 100;
-
- // (200) Routine information, such as ongoing status or performance.
- INFO = 200;
-
- // (300) Normal but significant events, such as start up, shut down, or
- // a configuration change.
- NOTICE = 300;
-
- // (400) Warning events might cause problems.
- WARNING = 400;
-
- // (500) Error events are likely to cause problems.
- ERROR = 500;
-
- // (600) Critical events cause more severe problems or outages.
- CRITICAL = 600;
-
- // (700) A person must take an action immediately.
- ALERT = 700;
-
- // (800) One or more systems are unusable.
- EMERGENCY = 800;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/longrunning/README.md b/contrib/libs/googleapis-common-protos/google/longrunning/README.md
deleted file mode 100644
index 7eba891ea0..0000000000
--- a/contrib/libs/googleapis-common-protos/google/longrunning/README.md
+++ /dev/null
@@ -1,31 +0,0 @@
-## Long-running Operations API
-
-This package contains the definition of an abstract interface that
-manages long running operations with API services.
-
-### Operation
-
-The primary message to understand within LRO is the `Operation` message.
-Operations have a unique name (in the context of a particular endpoint).
-Additionally, a service (called `Operations` -- plural) defines the interface
-for querying the state of any given operation.
-
-APIs that implement a concept of long-running operations are encouraged
-to follow this pattern: When a caller invokes an API backend to start a job...
-
- * The API backend starts asychronous work to fulfill the caller's
- request, and generates a unique name (the `Operation` name) to refer
- to the ongoing asychronous work.
- * The API backend immediately returns the `Operation` back to the caller.
- * The caller can invoke the API methods defined in the `Operations` service
- to get the current status of the asychronous work, and also to
- discover the final result (success or error).
-
-For Google APIs, the implementation of this pattern and the use of this
-proto are part of our [design rules][operations-rules]. Additionally, our
-[API client tooling][gapic-generator] seeks to be intelligent about these, to
-improve the client API consumption experience. Therefore, APIs outside of
-Google can also benefit by following this same pattern.
-
- [operations-rules]: https://cloud.google.com/apis/design/design_patterns#long_running_operations
- [gapic-generator]: https://github.com/googleapis/gapic-generator
diff --git a/contrib/libs/googleapis-common-protos/google/longrunning/operations.proto b/contrib/libs/googleapis-common-protos/google/longrunning/operations.proto
deleted file mode 100644
index 299eefb2e5..0000000000
--- a/contrib/libs/googleapis-common-protos/google/longrunning/operations.proto
+++ /dev/null
@@ -1,247 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.longrunning;
-
-import "google/api/annotations.proto";
-import "google/api/client.proto";
-import "google/protobuf/any.proto";
-import "google/protobuf/duration.proto";
-import "google/protobuf/empty.proto";
-import "google/rpc/status.proto";
-import "google/protobuf/descriptor.proto";
-
-option cc_enable_arenas = true;
-option csharp_namespace = "Google.LongRunning";
-option go_package = "google.golang.org/genproto/googleapis/longrunning;longrunning";
-option java_multiple_files = true;
-option java_outer_classname = "OperationsProto";
-option java_package = "com.google.longrunning";
-option php_namespace = "Google\\LongRunning";
-
-extend google.protobuf.MethodOptions {
- // Additional information regarding long-running operations.
- // In particular, this specifies the types that are returned from
- // long-running operations.
- //
- // Required for methods that return `google.longrunning.Operation`; invalid
- // otherwise.
- google.longrunning.OperationInfo operation_info = 1049;
-}
-
-// Manages long-running operations with an API service.
-//
-// When an API method normally takes long time to complete, it can be designed
-// to return [Operation][google.longrunning.Operation] to the client, and the client can use this
-// interface to receive the real response asynchronously by polling the
-// operation resource, or pass the operation resource to another API (such as
-// Google Cloud Pub/Sub API) to receive the response. Any API service that
-// returns long-running operations should implement the `Operations` interface
-// so developers can have a consistent client experience.
-service Operations {
- option (google.api.default_host) = "longrunning.googleapis.com";
-
- // Lists operations that match the specified filter in the request. If the
- // server doesn't support this method, it returns `UNIMPLEMENTED`.
- //
- // NOTE: the `name` binding allows API services to override the binding
- // to use different resource name schemes, such as `users/*/operations`. To
- // override the binding, API services can add a binding such as
- // `"/v1/{name=users/*}/operations"` to their service configuration.
- // For backwards compatibility, the default name includes the operations
- // collection id, however overriding users must ensure the name binding
- // is the parent resource, without the operations collection id.
- rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) {
- option (google.api.http) = {
- get: "/v1/{name=operations}"
- };
- option (google.api.method_signature) = "name,filter";
- }
-
- // Gets the latest state of a long-running operation. Clients can use this
- // method to poll the operation result at intervals as recommended by the API
- // service.
- rpc GetOperation(GetOperationRequest) returns (Operation) {
- option (google.api.http) = {
- get: "/v1/{name=operations/**}"
- };
- option (google.api.method_signature) = "name";
- }
-
- // Deletes a long-running operation. This method indicates that the client is
- // no longer interested in the operation result. It does not cancel the
- // operation. If the server doesn't support this method, it returns
- // `google.rpc.Code.UNIMPLEMENTED`.
- rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) {
- option (google.api.http) = {
- delete: "/v1/{name=operations/**}"
- };
- option (google.api.method_signature) = "name";
- }
-
- // Starts asynchronous cancellation on a long-running operation. The server
- // makes a best effort to cancel the operation, but success is not
- // guaranteed. If the server doesn't support this method, it returns
- // `google.rpc.Code.UNIMPLEMENTED`. Clients can use
- // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or
- // other methods to check whether the cancellation succeeded or whether the
- // operation completed despite cancellation. On successful cancellation,
- // the operation is not deleted; instead, it becomes an operation with
- // an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1,
- // corresponding to `Code.CANCELLED`.
- rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) {
- option (google.api.http) = {
- post: "/v1/{name=operations/**}:cancel"
- body: "*"
- };
- option (google.api.method_signature) = "name";
- }
-
- // Waits for the specified long-running operation until it is done or reaches
- // at most a specified timeout, returning the latest state. If the operation
- // is already done, the latest state is immediately returned. If the timeout
- // specified is greater than the default HTTP/RPC timeout, the HTTP/RPC
- // timeout is used. If the server does not support this method, it returns
- // `google.rpc.Code.UNIMPLEMENTED`.
- // Note that this method is on a best-effort basis. It may return the latest
- // state before the specified timeout (including immediately), meaning even an
- // immediate response is no guarantee that the operation is done.
- rpc WaitOperation(WaitOperationRequest) returns (Operation) {
- }
-}
-
-// This resource represents a long-running operation that is the result of a
-// network API call.
-message Operation {
- // The server-assigned name, which is only unique within the same service that
- // originally returns it. If you use the default HTTP mapping, the
- // `name` should be a resource name ending with `operations/{unique_id}`.
- string name = 1;
-
- // Service-specific metadata associated with the operation. It typically
- // contains progress information and common metadata such as create time.
- // Some services might not provide such metadata. Any method that returns a
- // long-running operation should document the metadata type, if any.
- google.protobuf.Any metadata = 2;
-
- // If the value is `false`, it means the operation is still in progress.
- // If `true`, the operation is completed, and either `error` or `response` is
- // available.
- bool done = 3;
-
- // The operation result, which can be either an `error` or a valid `response`.
- // If `done` == `false`, neither `error` nor `response` is set.
- // If `done` == `true`, exactly one of `error` or `response` is set.
- oneof result {
- // The error result of the operation in case of failure or cancellation.
- google.rpc.Status error = 4;
-
- // The normal response of the operation in case of success. If the original
- // method returns no data on success, such as `Delete`, the response is
- // `google.protobuf.Empty`. If the original method is standard
- // `Get`/`Create`/`Update`, the response should be the resource. For other
- // methods, the response should have the type `XxxResponse`, where `Xxx`
- // is the original method name. For example, if the original method name
- // is `TakeSnapshot()`, the inferred response type is
- // `TakeSnapshotResponse`.
- google.protobuf.Any response = 5;
- }
-}
-
-// The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation].
-message GetOperationRequest {
- // The name of the operation resource.
- string name = 1;
-}
-
-// The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations].
-message ListOperationsRequest {
- // The name of the operation's parent resource.
- string name = 4;
-
- // The standard list filter.
- string filter = 1;
-
- // The standard list page size.
- int32 page_size = 2;
-
- // The standard list page token.
- string page_token = 3;
-}
-
-// The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations].
-message ListOperationsResponse {
- // A list of operations that matches the specified filter in the request.
- repeated Operation operations = 1;
-
- // The standard List next-page token.
- string next_page_token = 2;
-}
-
-// The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation].
-message CancelOperationRequest {
- // The name of the operation resource to be cancelled.
- string name = 1;
-}
-
-// The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation].
-message DeleteOperationRequest {
- // The name of the operation resource to be deleted.
- string name = 1;
-}
-
-// The request message for [Operations.WaitOperation][google.longrunning.Operations.WaitOperation].
-message WaitOperationRequest {
- // The name of the operation resource to wait on.
- string name = 1;
-
- // The maximum duration to wait before timing out. If left blank, the wait
- // will be at most the time permitted by the underlying HTTP/RPC protocol.
- // If RPC context deadline is also specified, the shorter one will be used.
- google.protobuf.Duration timeout = 2;
-}
-
-// A message representing the message types used by a long-running operation.
-//
-// Example:
-//
-// rpc LongRunningRecognize(LongRunningRecognizeRequest)
-// returns (google.longrunning.Operation) {
-// option (google.longrunning.operation_info) = {
-// response_type: "LongRunningRecognizeResponse"
-// metadata_type: "LongRunningRecognizeMetadata"
-// };
-// }
-message OperationInfo {
- // Required. The message name of the primary return type for this
- // long-running operation.
- // This type will be used to deserialize the LRO's response.
- //
- // If the response is in a different package from the rpc, a fully-qualified
- // message name must be used (e.g. `google.protobuf.Struct`).
- //
- // Note: Altering this value constitutes a breaking change.
- string response_type = 1;
-
- // Required. The message name of the metadata type for this long-running
- // operation.
- //
- // If the response is in a different package from the rpc, a fully-qualified
- // message name must be used (e.g. `google.protobuf.Struct`).
- //
- // Note: Altering this value constitutes a breaking change.
- string metadata_type = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/rpc/README.md b/contrib/libs/googleapis-common-protos/google/rpc/README.md
deleted file mode 100644
index b47750076b..0000000000
--- a/contrib/libs/googleapis-common-protos/google/rpc/README.md
+++ /dev/null
@@ -1,18 +0,0 @@
-## RPC (Remote Procedure Call) Types
-
-This package contains [protocol buffer][protobuf] types that represent remote procedure
-call concepts. While [gRPC](https://grpc.io) uses these types, we encourage their
-use in any interested RPC implementation to promote compatibility and consistency.
-
-### Key Concepts
-
-- **Code**: An enum that represents an error code returned by an RPC. These error codes
- map to HTTP codes, but are slightly finer-grained. Every gRPC code has exactly one
- corresponding HTTP code; however, some HTTP codes have more than one corresponding
- gRPC code.
-- **Error details**: Any of the types contained in `error_details.proto` which provide
- extra details about particular types of failures.
-- **Status**: Combines a code, message, and error details to represent the success or
- failure details of an RPC call.
-
-[protobuf]: https://developers.google.com/protocol-buffers/
diff --git a/contrib/libs/googleapis-common-protos/google/rpc/code.proto b/contrib/libs/googleapis-common-protos/google/rpc/code.proto
deleted file mode 100644
index 98ae0ac18f..0000000000
--- a/contrib/libs/googleapis-common-protos/google/rpc/code.proto
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.rpc;
-
-option go_package = "google.golang.org/genproto/googleapis/rpc/code;code";
-option java_multiple_files = true;
-option java_outer_classname = "CodeProto";
-option java_package = "com.google.rpc";
-option objc_class_prefix = "RPC";
-
-// The canonical error codes for gRPC APIs.
-//
-//
-// Sometimes multiple error codes may apply. Services should return
-// the most specific error code that applies. For example, prefer
-// `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply.
-// Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`.
-enum Code {
- // Not an error; returned on success
- //
- // HTTP Mapping: 200 OK
- OK = 0;
-
- // The operation was cancelled, typically by the caller.
- //
- // HTTP Mapping: 499 Client Closed Request
- CANCELLED = 1;
-
- // Unknown error. For example, this error may be returned when
- // a `Status` value received from another address space belongs to
- // an error space that is not known in this address space. Also
- // errors raised by APIs that do not return enough error information
- // may be converted to this error.
- //
- // HTTP Mapping: 500 Internal Server Error
- UNKNOWN = 2;
-
- // The client specified an invalid argument. Note that this differs
- // from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments
- // that are problematic regardless of the state of the system
- // (e.g., a malformed file name).
- //
- // HTTP Mapping: 400 Bad Request
- INVALID_ARGUMENT = 3;
-
- // The deadline expired before the operation could complete. For operations
- // that change the state of the system, this error may be returned
- // even if the operation has completed successfully. For example, a
- // successful response from a server could have been delayed long
- // enough for the deadline to expire.
- //
- // HTTP Mapping: 504 Gateway Timeout
- DEADLINE_EXCEEDED = 4;
-
- // Some requested entity (e.g., file or directory) was not found.
- //
- // Note to server developers: if a request is denied for an entire class
- // of users, such as gradual feature rollout or undocumented whitelist,
- // `NOT_FOUND` may be used. If a request is denied for some users within
- // a class of users, such as user-based access control, `PERMISSION_DENIED`
- // must be used.
- //
- // HTTP Mapping: 404 Not Found
- NOT_FOUND = 5;
-
- // The entity that a client attempted to create (e.g., file or directory)
- // already exists.
- //
- // HTTP Mapping: 409 Conflict
- ALREADY_EXISTS = 6;
-
- // The caller does not have permission to execute the specified
- // operation. `PERMISSION_DENIED` must not be used for rejections
- // caused by exhausting some resource (use `RESOURCE_EXHAUSTED`
- // instead for those errors). `PERMISSION_DENIED` must not be
- // used if the caller can not be identified (use `UNAUTHENTICATED`
- // instead for those errors). This error code does not imply the
- // request is valid or the requested entity exists or satisfies
- // other pre-conditions.
- //
- // HTTP Mapping: 403 Forbidden
- PERMISSION_DENIED = 7;
-
- // The request does not have valid authentication credentials for the
- // operation.
- //
- // HTTP Mapping: 401 Unauthorized
- UNAUTHENTICATED = 16;
-
- // Some resource has been exhausted, perhaps a per-user quota, or
- // perhaps the entire file system is out of space.
- //
- // HTTP Mapping: 429 Too Many Requests
- RESOURCE_EXHAUSTED = 8;
-
- // The operation was rejected because the system is not in a state
- // required for the operation's execution. For example, the directory
- // to be deleted is non-empty, an rmdir operation is applied to
- // a non-directory, etc.
- //
- // Service implementors can use the following guidelines to decide
- // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`:
- // (a) Use `UNAVAILABLE` if the client can retry just the failing call.
- // (b) Use `ABORTED` if the client should retry at a higher level
- // (e.g., when a client-specified test-and-set fails, indicating the
- // client should restart a read-modify-write sequence).
- // (c) Use `FAILED_PRECONDITION` if the client should not retry until
- // the system state has been explicitly fixed. E.g., if an "rmdir"
- // fails because the directory is non-empty, `FAILED_PRECONDITION`
- // should be returned since the client should not retry unless
- // the files are deleted from the directory.
- //
- // HTTP Mapping: 400 Bad Request
- FAILED_PRECONDITION = 9;
-
- // The operation was aborted, typically due to a concurrency issue such as
- // a sequencer check failure or transaction abort.
- //
- // See the guidelines above for deciding between `FAILED_PRECONDITION`,
- // `ABORTED`, and `UNAVAILABLE`.
- //
- // HTTP Mapping: 409 Conflict
- ABORTED = 10;
-
- // The operation was attempted past the valid range. E.g., seeking or
- // reading past end-of-file.
- //
- // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may
- // be fixed if the system state changes. For example, a 32-bit file
- // system will generate `INVALID_ARGUMENT` if asked to read at an
- // offset that is not in the range [0,2^32-1], but it will generate
- // `OUT_OF_RANGE` if asked to read from an offset past the current
- // file size.
- //
- // There is a fair bit of overlap between `FAILED_PRECONDITION` and
- // `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific
- // error) when it applies so that callers who are iterating through
- // a space can easily look for an `OUT_OF_RANGE` error to detect when
- // they are done.
- //
- // HTTP Mapping: 400 Bad Request
- OUT_OF_RANGE = 11;
-
- // The operation is not implemented or is not supported/enabled in this
- // service.
- //
- // HTTP Mapping: 501 Not Implemented
- UNIMPLEMENTED = 12;
-
- // Internal errors. This means that some invariants expected by the
- // underlying system have been broken. This error code is reserved
- // for serious errors.
- //
- // HTTP Mapping: 500 Internal Server Error
- INTERNAL = 13;
-
- // The service is currently unavailable. This is most likely a
- // transient condition, which can be corrected by retrying with
- // a backoff. Note that it is not always safe to retry
- // non-idempotent operations.
- //
- // See the guidelines above for deciding between `FAILED_PRECONDITION`,
- // `ABORTED`, and `UNAVAILABLE`.
- //
- // HTTP Mapping: 503 Service Unavailable
- UNAVAILABLE = 14;
-
- // Unrecoverable data loss or corruption.
- //
- // HTTP Mapping: 500 Internal Server Error
- DATA_LOSS = 15;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/rpc/context/attribute_context.proto b/contrib/libs/googleapis-common-protos/google/rpc/context/attribute_context.proto
deleted file mode 100644
index e60a5ae94e..0000000000
--- a/contrib/libs/googleapis-common-protos/google/rpc/context/attribute_context.proto
+++ /dev/null
@@ -1,287 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.rpc.context;
-
-import "google/protobuf/struct.proto";
-import "google/protobuf/timestamp.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/rpc/context/attribute_context;attribute_context";
-option java_multiple_files = true;
-option java_outer_classname = "AttributeContextProto";
-option java_package = "com.google.rpc.context";
-
-// This message defines the standard attribute vocabulary for Google APIs.
-//
-// An attribute is a piece of metadata that describes an activity on a network
-// service. For example, the size of an HTTP request, or the status code of
-// an HTTP response.
-//
-// Each attribute has a type and a name, which is logically defined as
-// a proto message field in `AttributeContext`. The field type becomes the
-// attribute type, and the field path becomes the attribute name. For example,
-// the attribute `source.ip` maps to field `AttributeContext.source.ip`.
-//
-// This message definition is guaranteed not to have any wire breaking change.
-// So you can use it directly for passing attributes across different systems.
-//
-// NOTE: Different system may generate different subset of attributes. Please
-// verify the system specification before relying on an attribute generated
-// a system.
-message AttributeContext {
- // This message defines attributes for a node that handles a network request.
- // The node can be either a service or an application that sends, forwards,
- // or receives the request. Service peers should fill in
- // `principal` and `labels` as appropriate.
- message Peer {
- // The IP address of the peer.
- string ip = 1;
-
- // The network port of the peer.
- int64 port = 2;
-
- // The labels associated with the peer.
- map<string, string> labels = 6;
-
- // The identity of this peer. Similar to `Request.auth.principal`, but
- // relative to the peer instead of the request. For example, the
- // idenity associated with a load balancer that forwared the request.
- string principal = 7;
-
- // The CLDR country/region code associated with the above IP address.
- // If the IP address is private, the `region_code` should reflect the
- // physical location where this peer is running.
- string region_code = 8;
- }
-
- // This message defines attributes associated with API operations, such as
- // a network API request. The terminology is based on the conventions used
- // by Google APIs, Istio, and OpenAPI.
- message Api {
- // The API service name. It is a logical identifier for a networked API,
- // such as "pubsub.googleapis.com". The naming syntax depends on the
- // API management system being used for handling the request.
- string service = 1;
-
- // The API operation name. For gRPC requests, it is the fully qualified API
- // method name, such as "google.pubsub.v1.Publisher.Publish". For OpenAPI
- // requests, it is the `operationId`, such as "getPet".
- string operation = 2;
-
- // The API protocol used for sending the request, such as "http", "https",
- // "grpc", or "internal".
- string protocol = 3;
-
- // The API version associated with the API operation above, such as "v1" or
- // "v1alpha1".
- string version = 4;
- }
-
- // This message defines request authentication attributes. Terminology is
- // based on the JSON Web Token (JWT) standard, but the terms also
- // correlate to concepts in other standards.
- message Auth {
- // The authenticated principal. Reflects the issuer (`iss`) and subject
- // (`sub`) claims within a JWT. The issuer and subject should be `/`
- // delimited, with `/` percent-encoded within the subject fragment. For
- // Google accounts, the principal format is:
- // "https://accounts.google.com/{id}"
- string principal = 1;
-
- // The intended audience(s) for this authentication information. Reflects
- // the audience (`aud`) claim within a JWT. The audience
- // value(s) depends on the `issuer`, but typically include one or more of
- // the following pieces of information:
- //
- // * The services intended to receive the credential such as
- // ["pubsub.googleapis.com", "storage.googleapis.com"]
- // * A set of service-based scopes. For example,
- // ["https://www.googleapis.com/auth/cloud-platform"]
- // * The client id of an app, such as the Firebase project id for JWTs
- // from Firebase Auth.
- //
- // Consult the documentation for the credential issuer to determine the
- // information provided.
- repeated string audiences = 2;
-
- // The authorized presenter of the credential. Reflects the optional
- // Authorized Presenter (`azp`) claim within a JWT or the
- // OAuth client id. For example, a Google Cloud Platform client id looks
- // as follows: "123456789012.apps.googleusercontent.com".
- string presenter = 3;
-
- // Structured claims presented with the credential. JWTs include
- // `{key: value}` pairs for standard and private claims. The following
- // is a subset of the standard required and optional claims that would
- // typically be presented for a Google-based JWT:
- //
- // {'iss': 'accounts.google.com',
- // 'sub': '113289723416554971153',
- // 'aud': ['123456789012', 'pubsub.googleapis.com'],
- // 'azp': '123456789012.apps.googleusercontent.com',
- // 'email': 'jsmith@example.com',
- // 'iat': 1353601026,
- // 'exp': 1353604926}
- //
- // SAML assertions are similarly specified, but with an identity provider
- // dependent structure.
- google.protobuf.Struct claims = 4;
-
- // A list of access level resource names that allow resources to be
- // accessed by authenticated requester. It is part of Secure GCP processing
- // for the incoming request. An access level string has the format:
- // "//{api_service_name}/accessPolicies/{policy_id}/accessLevels/{short_name}"
- //
- // Example:
- // "//accesscontextmanager.googleapis.com/accessPolicies/MY_POLICY_ID/accessLevels/MY_LEVEL"
- repeated string access_levels = 5;
- }
-
- // This message defines attributes for an HTTP request. If the actual
- // request is not an HTTP request, the runtime system should try to map
- // the actual request to an equivalent HTTP request.
- message Request {
- // The unique ID for a request, which can be propagated to downstream
- // systems. The ID should have low probability of collision
- // within a single day for a specific service.
- string id = 1;
-
- // The HTTP request method, such as `GET`, `POST`.
- string method = 2;
-
- // The HTTP request headers. If multiple headers share the same key, they
- // must be merged according to the HTTP spec. All header keys must be
- // lowercased, because HTTP header keys are case-insensitive.
- map<string, string> headers = 3;
-
- // The HTTP URL path.
- string path = 4;
-
- // The HTTP request `Host` header value.
- string host = 5;
-
- // The HTTP URL scheme, such as `http` and `https`.
- string scheme = 6;
-
- // The HTTP URL query in the format of `name1=value1&name2=value2`, as it
- // appears in the first line of the HTTP request. No decoding is performed.
- string query = 7;
-
- // The timestamp when the `destination` service receives the first byte of
- // the request.
- google.protobuf.Timestamp time = 9;
-
- // The HTTP request size in bytes. If unknown, it must be -1.
- int64 size = 10;
-
- // The network protocol used with the request, such as "http/1.1",
- // "spdy/3", "h2", "h2c", "webrtc", "tcp", "udp", "quic". See
- // https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids
- // for details.
- string protocol = 11;
-
- // A special parameter for request reason. It is used by security systems
- // to associate auditing information with a request.
- string reason = 12;
-
- // The request authentication. May be absent for unauthenticated requests.
- // Derived from the HTTP request `Authorization` header or equivalent.
- Auth auth = 13;
- }
-
- // This message defines attributes for a typical network response. It
- // generally models semantics of an HTTP response.
- message Response {
- // The HTTP response status code, such as `200` and `404`.
- int64 code = 1;
-
- // The HTTP response size in bytes. If unknown, it must be -1.
- int64 size = 2;
-
- // The HTTP response headers. If multiple headers share the same key, they
- // must be merged according to HTTP spec. All header keys must be
- // lowercased, because HTTP header keys are case-insensitive.
- map<string, string> headers = 3;
-
- // The timestamp when the `destination` service generates the first byte of
- // the response.
- google.protobuf.Timestamp time = 4;
- }
-
- // This message defines core attributes for a resource. A resource is an
- // addressable (named) entity provided by the destination service. For
- // example, a file stored on a network storage service.
- message Resource {
- // The name of the service that this resource belongs to, such as
- // `pubsub.googleapis.com`. The service may be different from the DNS
- // hostname that actually serves the request.
- string service = 1;
-
- // The stable identifier (name) of a resource on the `service`. A resource
- // can be logically identified as "//{resource.service}/{resource.name}".
- // The differences between a resource name and a URI are:
- //
- // * Resource name is a logical identifier, independent of network
- // protocol and API version. For example,
- // `//pubsub.googleapis.com/projects/123/topics/news-feed`.
- // * URI often includes protocol and version information, so it can
- // be used directly by applications. For example,
- // `https://pubsub.googleapis.com/v1/projects/123/topics/news-feed`.
- //
- // See https://cloud.google.com/apis/design/resource_names for details.
- string name = 2;
-
- // The type of the resource. The syntax is platform-specific because
- // different platforms define their resources differently.
- //
- // For Google APIs, the type format must be "{service}/{kind}".
- string type = 3;
-
- // The labels or tags on the resource, such as AWS resource tags and
- // Kubernetes resource labels.
- map<string, string> labels = 4;
- }
-
- // The origin of a network activity. In a multi hop network activity,
- // the origin represents the sender of the first hop. For the first hop,
- // the `source` and the `origin` must have the same content.
- Peer origin = 7;
-
- // The source of a network activity, such as starting a TCP connection.
- // In a multi hop network activity, the source represents the sender of the
- // last hop.
- Peer source = 1;
-
- // The destination of a network activity, such as accepting a TCP connection.
- // In a multi hop network activity, the destination represents the receiver of
- // the last hop.
- Peer destination = 2;
-
- // Represents a network request, such as an HTTP request.
- Request request = 3;
-
- // Represents a network response, such as an HTTP response.
- Response response = 4;
-
- // Represents a target resource that is involved with a network activity.
- // If multiple resources are involved with an activity, this must be the
- // primary one.
- Resource resource = 5;
-
- // Represents an API operation that is involved to a network activity.
- Api api = 6;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/rpc/error_details.proto b/contrib/libs/googleapis-common-protos/google/rpc/error_details.proto
deleted file mode 100644
index 4862ebfe88..0000000000
--- a/contrib/libs/googleapis-common-protos/google/rpc/error_details.proto
+++ /dev/null
@@ -1,246 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.rpc;
-
-import "google/protobuf/duration.proto";
-
-option go_package = "google.golang.org/genproto/googleapis/rpc/errdetails;errdetails";
-option java_multiple_files = true;
-option java_outer_classname = "ErrorDetailsProto";
-option java_package = "com.google.rpc";
-option objc_class_prefix = "RPC";
-
-// Describes when the clients can retry a failed request. Clients could ignore
-// the recommendation here or retry when this information is missing from error
-// responses.
-//
-// It's always recommended that clients should use exponential backoff when
-// retrying.
-//
-// Clients should wait until `retry_delay` amount of time has passed since
-// receiving the error response before retrying. If retrying requests also
-// fail, clients should use an exponential backoff scheme to gradually increase
-// the delay between retries based on `retry_delay`, until either a maximum
-// number of retries have been reached or a maximum retry delay cap has been
-// reached.
-message RetryInfo {
- // Clients should wait at least this long between retrying the same request.
- google.protobuf.Duration retry_delay = 1;
-}
-
-// Describes additional debugging info.
-message DebugInfo {
- // The stack trace entries indicating where the error occurred.
- repeated string stack_entries = 1;
-
- // Additional debugging information provided by the server.
- string detail = 2;
-}
-
-// Describes how a quota check failed.
-//
-// For example if a daily limit was exceeded for the calling project,
-// a service could respond with a QuotaFailure detail containing the project
-// id and the description of the quota limit that was exceeded. If the
-// calling project hasn't enabled the service in the developer console, then
-// a service could respond with the project id and set `service_disabled`
-// to true.
-//
-// Also see RetryInfo and Help types for other details about handling a
-// quota failure.
-message QuotaFailure {
- // A message type used to describe a single quota violation. For example, a
- // daily quota or a custom quota that was exceeded.
- message Violation {
- // The subject on which the quota check failed.
- // For example, "clientip:<ip address of client>" or "project:<Google
- // developer project id>".
- string subject = 1;
-
- // A description of how the quota check failed. Clients can use this
- // description to find more about the quota configuration in the service's
- // public documentation, or find the relevant quota limit to adjust through
- // developer console.
- //
- // For example: "Service disabled" or "Daily Limit for read operations
- // exceeded".
- string description = 2;
- }
-
- // Describes all quota violations.
- repeated Violation violations = 1;
-}
-
-// Describes the cause of the error with structured details.
-//
-// Example of an error when contacting the "pubsub.googleapis.com" API when it
-// is not enabled:
-// { "reason": "API_DISABLED"
-// "domain": "googleapis.com"
-// "metadata": {
-// "resource": "projects/123",
-// "service": "pubsub.googleapis.com"
-// }
-// }
-// This response indicates that the pubsub.googleapis.com API is not enabled.
-//
-// Example of an error that is returned when attempting to create a Spanner
-// instance in a region that is out of stock:
-// { "reason": "STOCKOUT"
-// "domain": "spanner.googleapis.com",
-// "metadata": {
-// "availableRegions": "us-central1,us-east2"
-// }
-// }
-//
-message ErrorInfo {
- // The reason of the error. This is a constant value that identifies the
- // proximate cause of the error. Error reasons are unique within a particular
- // domain of errors. This should be at most 63 characters and match
- // /[A-Z0-9_]+/.
- string reason = 1;
-
- // The logical grouping to which the "reason" belongs. Often "domain" will
- // contain the registered service name of the tool or product that is the
- // source of the error. Example: "pubsub.googleapis.com". If the error is
- // common across many APIs, the first segment of the example above will be
- // omitted. The value will be, "googleapis.com".
- string domain = 2;
-
- // Additional structured details about this error.
- //
- // Keys should match /[a-zA-Z0-9-_]/ and be limited to 64 characters in
- // length. When identifying the current value of an exceeded limit, the units
- // should be contained in the key, not the value. For example, rather than
- // {"instanceLimit": "100/request"}, should be returned as,
- // {"instanceLimitPerRequest": "100"}, if the client exceeds the number of
- // instances that can be created in a single (batch) request.
- map<string, string> metadata = 3;
-}
-
-// Describes what preconditions have failed.
-//
-// For example, if an RPC failed because it required the Terms of Service to be
-// acknowledged, it could list the terms of service violation in the
-// PreconditionFailure message.
-message PreconditionFailure {
- // A message type used to describe a single precondition failure.
- message Violation {
- // The type of PreconditionFailure. We recommend using a service-specific
- // enum type to define the supported precondition violation subjects. For
- // example, "TOS" for "Terms of Service violation".
- string type = 1;
-
- // The subject, relative to the type, that failed.
- // For example, "google.com/cloud" relative to the "TOS" type would indicate
- // which terms of service is being referenced.
- string subject = 2;
-
- // A description of how the precondition failed. Developers can use this
- // description to understand how to fix the failure.
- //
- // For example: "Terms of service not accepted".
- string description = 3;
- }
-
- // Describes all precondition violations.
- repeated Violation violations = 1;
-}
-
-// Describes violations in a client request. This error type focuses on the
-// syntactic aspects of the request.
-message BadRequest {
- // A message type used to describe a single bad request field.
- message FieldViolation {
- // A path leading to a field in the request body. The value will be a
- // sequence of dot-separated identifiers that identify a protocol buffer
- // field. E.g., "field_violations.field" would identify this field.
- string field = 1;
-
- // A description of why the request element is bad.
- string description = 2;
- }
-
- // Describes all violations in a client request.
- repeated FieldViolation field_violations = 1;
-}
-
-// Contains metadata about the request that clients can attach when filing a bug
-// or providing other forms of feedback.
-message RequestInfo {
- // An opaque string that should only be interpreted by the service generating
- // it. For example, it can be used to identify requests in the service's logs.
- string request_id = 1;
-
- // Any data that was used to serve this request. For example, an encrypted
- // stack trace that can be sent back to the service provider for debugging.
- string serving_data = 2;
-}
-
-// Describes the resource that is being accessed.
-message ResourceInfo {
- // A name for the type of resource being accessed, e.g. "sql table",
- // "cloud storage bucket", "file", "Google calendar"; or the type URL
- // of the resource: e.g. "type.googleapis.com/google.pubsub.v1.Topic".
- string resource_type = 1;
-
- // The name of the resource being accessed. For example, a shared calendar
- // name: "example.com_4fghdhgsrgh@group.calendar.google.com", if the current
- // error is [google.rpc.Code.PERMISSION_DENIED][google.rpc.Code.PERMISSION_DENIED].
- string resource_name = 2;
-
- // The owner of the resource (optional).
- // For example, "user:<owner email>" or "project:<Google developer project
- // id>".
- string owner = 3;
-
- // Describes what error is encountered when accessing this resource.
- // For example, updating a cloud project may require the `writer` permission
- // on the developer console project.
- string description = 4;
-}
-
-// Provides links to documentation or for performing an out of band action.
-//
-// For example, if a quota check failed with an error indicating the calling
-// project hasn't enabled the accessed service, this can contain a URL pointing
-// directly to the right place in the developer console to flip the bit.
-message Help {
- // Describes a URL link.
- message Link {
- // Describes what the link offers.
- string description = 1;
-
- // The URL of the link.
- string url = 2;
- }
-
- // URL(s) pointing to additional information on handling the current error.
- repeated Link links = 1;
-}
-
-// Provides a localized error message that is safe to return to the user
-// which can be attached to an RPC error.
-message LocalizedMessage {
- // The locale used following the specification defined at
- // http://www.rfc-editor.org/rfc/bcp/bcp47.txt.
- // Examples are: "en-US", "fr-CH", "es-MX"
- string locale = 1;
-
- // The localized error message in the above locale.
- string message = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/rpc/status.proto b/contrib/libs/googleapis-common-protos/google/rpc/status.proto
deleted file mode 100644
index 3b1f7a932f..0000000000
--- a/contrib/libs/googleapis-common-protos/google/rpc/status.proto
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.rpc;
-
-import "google/protobuf/any.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/rpc/status;status";
-option java_multiple_files = true;
-option java_outer_classname = "StatusProto";
-option java_package = "com.google.rpc";
-option objc_class_prefix = "RPC";
-
-// The `Status` type defines a logical error model that is suitable for
-// different programming environments, including REST APIs and RPC APIs. It is
-// used by [gRPC](https://github.com/grpc). Each `Status` message contains
-// three pieces of data: error code, error message, and error details.
-//
-// You can find out more about this error model and how to work with it in the
-// [API Design Guide](https://cloud.google.com/apis/design/errors).
-message Status {
- // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code].
- int32 code = 1;
-
- // A developer-facing error message, which should be in English. Any
- // user-facing error message should be localized and sent in the
- // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client.
- string message = 2;
-
- // A list of messages that carry the error details. There is a common set of
- // message types for APIs to use.
- repeated google.protobuf.Any details = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/README.md b/contrib/libs/googleapis-common-protos/google/type/README.md
deleted file mode 100644
index adf1563a82..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-## Google Common Types
-
-This package contains definitions of common types for Google APIs.
-All types defined in this package are suitable for different APIs to
-exchange data, and will never break binary compatibility. They should
-have design quality comparable to major programming languages like
-Java and C#.
diff --git a/contrib/libs/googleapis-common-protos/google/type/calendar_period.proto b/contrib/libs/googleapis-common-protos/google/type/calendar_period.proto
deleted file mode 100644
index a91d0c35c8..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/calendar_period.proto
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod";
-option java_multiple_files = true;
-option java_outer_classname = "CalendarPeriodProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// A `CalendarPeriod` represents the abstract concept of a time period that has
-// a canonical start. Grammatically, "the start of the current
-// `CalendarPeriod`." All calendar times begin at midnight UTC.
-enum CalendarPeriod {
- // Undefined period, raises an error.
- CALENDAR_PERIOD_UNSPECIFIED = 0;
-
- // A day.
- DAY = 1;
-
- // A week. Weeks begin on Monday, following
- // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
- WEEK = 2;
-
- // A fortnight. The first calendar fortnight of the year begins at the start
- // of week 1 according to
- // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
- FORTNIGHT = 3;
-
- // A month.
- MONTH = 4;
-
- // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each
- // year.
- QUARTER = 5;
-
- // A half-year. Half-years start on dates 1-Jan and 1-Jul.
- HALF = 6;
-
- // A year.
- YEAR = 7;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/color.proto b/contrib/libs/googleapis-common-protos/google/type/color.proto
deleted file mode 100644
index 417f1c4b19..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/color.proto
+++ /dev/null
@@ -1,170 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-import "google/protobuf/wrappers.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/color;color";
-option java_multiple_files = true;
-option java_outer_classname = "ColorProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a color in the RGBA color space. This representation is designed
-// for simplicity of conversion to/from color representations in various
-// languages over compactness; for example, the fields of this representation
-// can be trivially provided to the constructor of "java.awt.Color" in Java; it
-// can also be trivially provided to UIColor's "+colorWithRed:green:blue:alpha"
-// method in iOS; and, with just a little work, it can be easily formatted into
-// a CSS "rgba()" string in JavaScript, as well.
-//
-// Note: this proto does not carry information about the absolute color space
-// that should be used to interpret the RGB value (e.g. sRGB, Adobe RGB,
-// DCI-P3, BT.2020, etc.). By default, applications SHOULD assume the sRGB color
-// space.
-//
-// Example (Java):
-//
-// import com.google.type.Color;
-//
-// // ...
-// public static java.awt.Color fromProto(Color protocolor) {
-// float alpha = protocolor.hasAlpha()
-// ? protocolor.getAlpha().getValue()
-// : 1.0;
-//
-// return new java.awt.Color(
-// protocolor.getRed(),
-// protocolor.getGreen(),
-// protocolor.getBlue(),
-// alpha);
-// }
-//
-// public static Color toProto(java.awt.Color color) {
-// float red = (float) color.getRed();
-// float green = (float) color.getGreen();
-// float blue = (float) color.getBlue();
-// float denominator = 255.0;
-// Color.Builder resultBuilder =
-// Color
-// .newBuilder()
-// .setRed(red / denominator)
-// .setGreen(green / denominator)
-// .setBlue(blue / denominator);
-// int alpha = color.getAlpha();
-// if (alpha != 255) {
-// result.setAlpha(
-// FloatValue
-// .newBuilder()
-// .setValue(((float) alpha) / denominator)
-// .build());
-// }
-// return resultBuilder.build();
-// }
-// // ...
-//
-// Example (iOS / Obj-C):
-//
-// // ...
-// static UIColor* fromProto(Color* protocolor) {
-// float red = [protocolor red];
-// float green = [protocolor green];
-// float blue = [protocolor blue];
-// FloatValue* alpha_wrapper = [protocolor alpha];
-// float alpha = 1.0;
-// if (alpha_wrapper != nil) {
-// alpha = [alpha_wrapper value];
-// }
-// return [UIColor colorWithRed:red green:green blue:blue alpha:alpha];
-// }
-//
-// static Color* toProto(UIColor* color) {
-// CGFloat red, green, blue, alpha;
-// if (![color getRed:&red green:&green blue:&blue alpha:&alpha]) {
-// return nil;
-// }
-// Color* result = [[Color alloc] init];
-// [result setRed:red];
-// [result setGreen:green];
-// [result setBlue:blue];
-// if (alpha <= 0.9999) {
-// [result setAlpha:floatWrapperWithValue(alpha)];
-// }
-// [result autorelease];
-// return result;
-// }
-// // ...
-//
-// Example (JavaScript):
-//
-// // ...
-//
-// var protoToCssColor = function(rgb_color) {
-// var redFrac = rgb_color.red || 0.0;
-// var greenFrac = rgb_color.green || 0.0;
-// var blueFrac = rgb_color.blue || 0.0;
-// var red = Math.floor(redFrac * 255);
-// var green = Math.floor(greenFrac * 255);
-// var blue = Math.floor(blueFrac * 255);
-//
-// if (!('alpha' in rgb_color)) {
-// return rgbToCssColor_(red, green, blue);
-// }
-//
-// var alphaFrac = rgb_color.alpha.value || 0.0;
-// var rgbParams = [red, green, blue].join(',');
-// return ['rgba(', rgbParams, ',', alphaFrac, ')'].join('');
-// };
-//
-// var rgbToCssColor_ = function(red, green, blue) {
-// var rgbNumber = new Number((red << 16) | (green << 8) | blue);
-// var hexString = rgbNumber.toString(16);
-// var missingZeros = 6 - hexString.length;
-// var resultBuilder = ['#'];
-// for (var i = 0; i < missingZeros; i++) {
-// resultBuilder.push('0');
-// }
-// resultBuilder.push(hexString);
-// return resultBuilder.join('');
-// };
-//
-// // ...
-message Color {
- // The amount of red in the color as a value in the interval [0, 1].
- float red = 1;
-
- // The amount of green in the color as a value in the interval [0, 1].
- float green = 2;
-
- // The amount of blue in the color as a value in the interval [0, 1].
- float blue = 3;
-
- // The fraction of this color that should be applied to the pixel. That is,
- // the final pixel color is defined by the equation:
- //
- // pixel color = alpha * (this color) + (1.0 - alpha) * (background color)
- //
- // This means that a value of 1.0 corresponds to a solid color, whereas
- // a value of 0.0 corresponds to a completely transparent color. This
- // uses a wrapper message rather than a simple float scalar so that it is
- // possible to distinguish between a default value and the value being unset.
- // If omitted, this color object is to be rendered as a solid color
- // (as if the alpha value had been explicitly given with a value of 1.0).
- google.protobuf.FloatValue alpha = 4;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/date.proto b/contrib/libs/googleapis-common-protos/google/type/date.proto
deleted file mode 100644
index b958feeba5..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/date.proto
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/date;date";
-option java_multiple_files = true;
-option java_outer_classname = "DateProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a whole or partial calendar date, e.g. a birthday. The time of day
-// and time zone are either specified elsewhere or are not significant. The date
-// is relative to the Proleptic Gregorian Calendar. This can represent:
-//
-// * A full date, with non-zero year, month and day values
-// * A month and day value, with a zero year, e.g. an anniversary
-// * A year on its own, with zero month and day values
-// * A year and month value, with a zero day, e.g. a credit card expiration date
-//
-// Related types are [google.type.TimeOfDay][google.type.TimeOfDay] and `google.protobuf.Timestamp`.
-message Date {
- // Year of date. Must be from 1 to 9999, or 0 if specifying a date without
- // a year.
- int32 year = 1;
-
- // Month of year. Must be from 1 to 12, or 0 if specifying a year without a
- // month and day.
- int32 month = 2;
-
- // Day of month. Must be from 1 to 31 and valid for the year and month, or 0
- // if specifying a year by itself or a year and month where the day is not
- // significant.
- int32 day = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/datetime.proto b/contrib/libs/googleapis-common-protos/google/type/datetime.proto
deleted file mode 100644
index 5aebc4b97f..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/datetime.proto
+++ /dev/null
@@ -1,97 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-import "google/protobuf/duration.proto";
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/datetime;datetime";
-option java_multiple_files = true;
-option java_outer_classname = "DateTimeProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents civil time in one of a few possible ways:
-//
-// * When utc_offset is set and time_zone is unset: a civil time on a calendar
-// day with a particular offset from UTC.
-// * When time_zone is set and utc_offset is unset: a civil time on a calendar
-// day in a particular time zone.
-// * When neither time_zone nor utc_offset is set: a civil time on a calendar
-// day in local time.
-//
-// The date is relative to the Proleptic Gregorian Calendar.
-//
-// If year is 0, the DateTime is considered not to have a specific year. month
-// and day must have valid, non-zero values.
-//
-// This type is more flexible than some applications may want. Make sure to
-// document and validate your application's limitations.
-message DateTime {
- // Optional. Year of date. Must be from 1 to 9999, or 0 if specifying a
- // datetime without a year.
- int32 year = 1;
-
- // Required. Month of year. Must be from 1 to 12.
- int32 month = 2;
-
- // Required. Day of month. Must be from 1 to 31 and valid for the year and
- // month.
- int32 day = 3;
-
- // Required. Hours of day in 24 hour format. Should be from 0 to 23. An API
- // may choose to allow the value "24:00:00" for scenarios like business
- // closing time.
- int32 hours = 4;
-
- // Required. Minutes of hour of day. Must be from 0 to 59.
- int32 minutes = 5;
-
- // Required. Seconds of minutes of the time. Must normally be from 0 to 59. An
- // API may allow the value 60 if it allows leap-seconds.
- int32 seconds = 6;
-
- // Required. Fractions of seconds in nanoseconds. Must be from 0 to
- // 999,999,999.
- int32 nanos = 7;
-
- // Optional. Specifies either the UTC offset or the time zone of the DateTime.
- // Choose carefully between them, considering that time zone data may change
- // in the future (for example, a country modifies their DST start/end dates,
- // and future DateTimes in the affected range had already been stored).
- // If omitted, the DateTime is considered to be in local time.
- oneof time_offset {
- // UTC offset. Must be whole seconds, between -18 hours and +18 hours.
- // For example, a UTC offset of -4:00 would be represented as
- // { seconds: -14400 }.
- google.protobuf.Duration utc_offset = 8;
-
- // Time zone.
- TimeZone time_zone = 9;
- }
-}
-
-// Represents a time zone from the
-// [IANA Time Zone Database](https://www.iana.org/time-zones).
-message TimeZone {
- // IANA Time Zone Database time zone, e.g. "America/New_York".
- string id = 1;
-
- // Optional. IANA Time Zone Database version number, e.g. "2019a".
- string version = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/dayofweek.proto b/contrib/libs/googleapis-common-protos/google/type/dayofweek.proto
deleted file mode 100644
index 7544e15161..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/dayofweek.proto
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option go_package = "google.golang.org/genproto/googleapis/type/dayofweek;dayofweek";
-option java_multiple_files = true;
-option java_outer_classname = "DayOfWeekProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a day of week.
-enum DayOfWeek {
- // The unspecified day-of-week.
- DAY_OF_WEEK_UNSPECIFIED = 0;
-
- // The day-of-week of Monday.
- MONDAY = 1;
-
- // The day-of-week of Tuesday.
- TUESDAY = 2;
-
- // The day-of-week of Wednesday.
- WEDNESDAY = 3;
-
- // The day-of-week of Thursday.
- THURSDAY = 4;
-
- // The day-of-week of Friday.
- FRIDAY = 5;
-
- // The day-of-week of Saturday.
- SATURDAY = 6;
-
- // The day-of-week of Sunday.
- SUNDAY = 7;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/expr.proto b/contrib/libs/googleapis-common-protos/google/type/expr.proto
deleted file mode 100644
index 5d4f2f71b2..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/expr.proto
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option go_package = "google.golang.org/genproto/googleapis/type/expr;expr";
-option java_multiple_files = true;
-option java_outer_classname = "ExprProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents an expression text. Example:
-//
-// title: "User account presence"
-// description: "Determines whether the request has a user account"
-// expression: "size(request.user) > 0"
-message Expr {
- // Textual representation of an expression in
- // Common Expression Language syntax.
- //
- // The application context of the containing message determines which
- // well-known feature set of CEL is supported.
- string expression = 1;
-
- // An optional title for the expression, i.e. a short string describing
- // its purpose. This can be used e.g. in UIs which allow to enter the
- // expression.
- string title = 2;
-
- // An optional description of the expression. This is a longer text which
- // describes the expression, e.g. when hovered over it in a UI.
- string description = 3;
-
- // An optional string indicating the location of the expression for error
- // reporting, e.g. a file name and a position in the file.
- string location = 4;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/fraction.proto b/contrib/libs/googleapis-common-protos/google/type/fraction.proto
deleted file mode 100644
index 8ad008dda2..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/fraction.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option go_package = "google.golang.org/genproto/googleapis/type/fraction;fraction";
-option java_multiple_files = true;
-option java_outer_classname = "FractionProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a fraction in terms of a numerator divided by a denominator.
-message Fraction {
- // The portion of the denominator in the faction, e.g. 2 in 2/3.
- int64 numerator = 1;
-
- // The value by which the numerator is divided, e.g. 3 in 2/3. Must be
- // positive.
- int64 denominator = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/latlng.proto b/contrib/libs/googleapis-common-protos/google/type/latlng.proto
deleted file mode 100644
index a90b7c23d0..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/latlng.proto
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/latlng;latlng";
-option java_multiple_files = true;
-option java_outer_classname = "LatLngProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// An object that represents a latitude/longitude pair. This is expressed as a
-// pair of doubles to represent degrees latitude and degrees longitude. Unless
-// specified otherwise, this must conform to the
-// <a href="http://www.unoosa.org/pdf/icg/2012/template/WGS_84.pdf">WGS84
-// standard</a>. Values must be within normalized ranges.
-message LatLng {
- // The latitude in degrees. It must be in the range [-90.0, +90.0].
- double latitude = 1;
-
- // The longitude in degrees. It must be in the range [-180.0, +180.0].
- double longitude = 2;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/money.proto b/contrib/libs/googleapis-common-protos/google/type/money.proto
deleted file mode 100644
index ef41f1089a..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/money.proto
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/money;money";
-option java_multiple_files = true;
-option java_outer_classname = "MoneyProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents an amount of money with its currency type.
-message Money {
- // The 3-letter currency code defined in ISO 4217.
- string currency_code = 1;
-
- // The whole units of the amount.
- // For example if `currencyCode` is `"USD"`, then 1 unit is one US dollar.
- int64 units = 2;
-
- // Number of nano (10^-9) units of the amount.
- // The value must be between -999,999,999 and +999,999,999 inclusive.
- // If `units` is positive, `nanos` must be positive or zero.
- // If `units` is zero, `nanos` can be positive, zero, or negative.
- // If `units` is negative, `nanos` must be negative or zero.
- // For example $-1.75 is represented as `units`=-1 and `nanos`=-750,000,000.
- int32 nanos = 3;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/month.proto b/contrib/libs/googleapis-common-protos/google/type/month.proto
deleted file mode 100644
index 54b7865f4b..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/month.proto
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option go_package = "google.golang.org/genproto/googleapis/type/month;month";
-option java_multiple_files = true;
-option java_outer_classname = "MonthProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a month in the Gregorian calendar.
-enum Month {
- // The unspecifed month.
- MONTH_UNSPECIFIED = 0;
-
- // The month of January.
- JANUARY = 1;
-
- // The month of February.
- FEBRUARY = 2;
-
- // The month of March.
- MARCH = 3;
-
- // The month of April.
- APRIL = 4;
-
- // The month of May.
- MAY = 5;
-
- // The month of June.
- JUNE = 6;
-
- // The month of July.
- JULY = 7;
-
- // The month of August.
- AUGUST = 8;
-
- // The month of September.
- SEPTEMBER = 9;
-
- // The month of October.
- OCTOBER = 10;
-
- // The month of November.
- NOVEMBER = 11;
-
- // The month of December.
- DECEMBER = 12;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/postal_address.proto b/contrib/libs/googleapis-common-protos/google/type/postal_address.proto
deleted file mode 100644
index 688af8a1bc..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/postal_address.proto
+++ /dev/null
@@ -1,135 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/postaladdress;postaladdress";
-option java_multiple_files = true;
-option java_outer_classname = "PostalAddressProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a postal address, e.g. for postal delivery or payments addresses.
-// Given a postal address, a postal service can deliver items to a premise, P.O.
-// Box or similar.
-// It is not intended to model geographical locations (roads, towns,
-// mountains).
-//
-// In typical usage an address would be created via user input or from importing
-// existing data, depending on the type of process.
-//
-// Advice on address input / editing:
-// - Use an i18n-ready address widget such as
-// https://github.com/google/libaddressinput)
-// - Users should not be presented with UI elements for input or editing of
-// fields outside countries where that field is used.
-//
-// For more guidance on how to use this schema, please see:
-// https://support.google.com/business/answer/6397478
-message PostalAddress {
- // The schema revision of the `PostalAddress`. This must be set to 0, which is
- // the latest revision.
- //
- // All new revisions **must** be backward compatible with old revisions.
- int32 revision = 1;
-
- // Required. CLDR region code of the country/region of the address. This
- // is never inferred and it is up to the user to ensure the value is
- // correct. See http://cldr.unicode.org/ and
- // http://www.unicode.org/cldr/charts/30/supplemental/territory_information.html
- // for details. Example: "CH" for Switzerland.
- string region_code = 2;
-
- // Optional. BCP-47 language code of the contents of this address (if
- // known). This is often the UI language of the input form or is expected
- // to match one of the languages used in the address' country/region, or their
- // transliterated equivalents.
- // This can affect formatting in certain countries, but is not critical
- // to the correctness of the data and will never affect any validation or
- // other non-formatting related operations.
- //
- // If this value is not known, it should be omitted (rather than specifying a
- // possibly incorrect default).
- //
- // Examples: "zh-Hant", "ja", "ja-Latn", "en".
- string language_code = 3;
-
- // Optional. Postal code of the address. Not all countries use or require
- // postal codes to be present, but where they are used, they may trigger
- // additional validation with other parts of the address (e.g. state/zip
- // validation in the U.S.A.).
- string postal_code = 4;
-
- // Optional. Additional, country-specific, sorting code. This is not used
- // in most regions. Where it is used, the value is either a string like
- // "CEDEX", optionally followed by a number (e.g. "CEDEX 7"), or just a number
- // alone, representing the "sector code" (Jamaica), "delivery area indicator"
- // (Malawi) or "post office indicator" (e.g. Côte d'Ivoire).
- string sorting_code = 5;
-
- // Optional. Highest administrative subdivision which is used for postal
- // addresses of a country or region.
- // For example, this can be a state, a province, an oblast, or a prefecture.
- // Specifically, for Spain this is the province and not the autonomous
- // community (e.g. "Barcelona" and not "Catalonia").
- // Many countries don't use an administrative area in postal addresses. E.g.
- // in Switzerland this should be left unpopulated.
- string administrative_area = 6;
-
- // Optional. Generally refers to the city/town portion of the address.
- // Examples: US city, IT comune, UK post town.
- // In regions of the world where localities are not well defined or do not fit
- // into this structure well, leave locality empty and use address_lines.
- string locality = 7;
-
- // Optional. Sublocality of the address.
- // For example, this can be neighborhoods, boroughs, districts.
- string sublocality = 8;
-
- // Unstructured address lines describing the lower levels of an address.
- //
- // Because values in address_lines do not have type information and may
- // sometimes contain multiple values in a single field (e.g.
- // "Austin, TX"), it is important that the line order is clear. The order of
- // address lines should be "envelope order" for the country/region of the
- // address. In places where this can vary (e.g. Japan), address_language is
- // used to make it explicit (e.g. "ja" for large-to-small ordering and
- // "ja-Latn" or "en" for small-to-large). This way, the most specific line of
- // an address can be selected based on the language.
- //
- // The minimum permitted structural representation of an address consists
- // of a region_code with all remaining information placed in the
- // address_lines. It would be possible to format such an address very
- // approximately without geocoding, but no semantic reasoning could be
- // made about any of the address components until it was at least
- // partially resolved.
- //
- // Creating an address only containing a region_code and address_lines, and
- // then geocoding is the recommended way to handle completely unstructured
- // addresses (as opposed to guessing which parts of the address should be
- // localities or administrative areas).
- repeated string address_lines = 9;
-
- // Optional. The recipient at the address.
- // This field may, under certain circumstances, contain multiline information.
- // For example, it might contain "care of" information.
- repeated string recipients = 10;
-
- // Optional. The name of the organization at the address.
- string organization = 11;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/quaternion.proto b/contrib/libs/googleapis-common-protos/google/type/quaternion.proto
deleted file mode 100644
index 7ab5dc7283..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/quaternion.proto
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/quaternion;quaternion";
-option java_multiple_files = true;
-option java_outer_classname = "QuaternionProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// A quaternion is defined as the quotient of two directed lines in a
-// three-dimensional space or equivalently as the quotient of two Euclidean
-// vectors (https://en.wikipedia.org/wiki/Quaternion).
-//
-// Quaternions are often used in calculations involving three-dimensional
-// rotations (https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation),
-// as they provide greater mathematical robustness by avoiding the gimbal lock
-// problems that can be encountered when using Euler angles
-// (https://en.wikipedia.org/wiki/Gimbal_lock).
-//
-// Quaternions are generally represented in this form:
-//
-// w + xi + yj + zk
-//
-// where x, y, z, and w are real numbers, and i, j, and k are three imaginary
-// numbers.
-//
-// Our naming choice `(x, y, z, w)` comes from the desire to avoid confusion for
-// those interested in the geometric properties of the quaternion in the 3D
-// Cartesian space. Other texts often use alternative names or subscripts, such
-// as `(a, b, c, d)`, `(1, i, j, k)`, or `(0, 1, 2, 3)`, which are perhaps
-// better suited for mathematical interpretations.
-//
-// To avoid any confusion, as well as to maintain compatibility with a large
-// number of software libraries, the quaternions represented using the protocol
-// buffer below *must* follow the Hamilton convention, which defines `ij = k`
-// (i.e. a right-handed algebra), and therefore:
-//
-// i^2 = j^2 = k^2 = ijk = −1
-// ij = −ji = k
-// jk = −kj = i
-// ki = −ik = j
-//
-// Please DO NOT use this to represent quaternions that follow the JPL
-// convention, or any of the other quaternion flavors out there.
-//
-// Definitions:
-//
-// - Quaternion norm (or magnitude): `sqrt(x^2 + y^2 + z^2 + w^2)`.
-// - Unit (or normalized) quaternion: a quaternion whose norm is 1.
-// - Pure quaternion: a quaternion whose scalar component (`w`) is 0.
-// - Rotation quaternion: a unit quaternion used to represent rotation.
-// - Orientation quaternion: a unit quaternion used to represent orientation.
-//
-// A quaternion can be normalized by dividing it by its norm. The resulting
-// quaternion maintains the same direction, but has a norm of 1, i.e. it moves
-// on the unit sphere. This is generally necessary for rotation and orientation
-// quaternions, to avoid rounding errors:
-// https://en.wikipedia.org/wiki/Rotation_formalisms_in_three_dimensions
-//
-// Note that `(x, y, z, w)` and `(-x, -y, -z, -w)` represent the same rotation,
-// but normalization would be even more useful, e.g. for comparison purposes, if
-// it would produce a unique representation. It is thus recommended that `w` be
-// kept positive, which can be achieved by changing all the signs when `w` is
-// negative.
-//
-message Quaternion {
- // The x component.
- double x = 1;
-
- // The y component.
- double y = 2;
-
- // The z component.
- double z = 3;
-
- // The scalar component.
- double w = 4;
-}
diff --git a/contrib/libs/googleapis-common-protos/google/type/timeofday.proto b/contrib/libs/googleapis-common-protos/google/type/timeofday.proto
deleted file mode 100644
index b609a48798..0000000000
--- a/contrib/libs/googleapis-common-protos/google/type/timeofday.proto
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2019 Google LLC.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-syntax = "proto3";
-
-package google.type;
-
-option cc_enable_arenas = true;
-option go_package = "google.golang.org/genproto/googleapis/type/timeofday;timeofday";
-option java_multiple_files = true;
-option java_outer_classname = "TimeOfDayProto";
-option java_package = "com.google.type";
-option objc_class_prefix = "GTP";
-
-// Represents a time of day. The date and time zone are either not significant
-// or are specified elsewhere. An API may choose to allow leap seconds. Related
-// types are [google.type.Date][google.type.Date] and `google.protobuf.Timestamp`.
-message TimeOfDay {
- // Hours of day in 24 hour format. Should be from 0 to 23. An API may choose
- // to allow the value "24:00:00" for scenarios like business closing time.
- int32 hours = 1;
-
- // Minutes of hour of day. Must be from 0 to 59.
- int32 minutes = 2;
-
- // Seconds of minutes of the time. Must normally be from 0 to 59. An API may
- // allow the value 60 if it allows leap-seconds.
- int32 seconds = 3;
-
- // Fractions of seconds in nanoseconds. Must be from 0 to 999,999,999.
- int32 nanos = 4;
-}
diff --git a/contrib/libs/minilzo/.yandex_meta/devtools.copyrights.report b/contrib/libs/minilzo/.yandex_meta/devtools.copyrights.report
deleted file mode 100644
index 9d6a71bbe1..0000000000
--- a/contrib/libs/minilzo/.yandex_meta/devtools.copyrights.report
+++ /dev/null
@@ -1,59 +0,0 @@
-# File format ($ symbol means the beginning of a line):
-#
-# $ # this message
-# $ # =======================
-# $ # comments (all commentaries should starts with some number of spaces and # symbol)
-# ${action} {license id} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make
-# ${all_file_action} filename
-# $ # user commentaries (many lines)
-# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/3/ya.make
-# ${all_file_action} filename
-# $ # user commentaries
-# $ generated description
-# $ ...
-#
-# You can modify action, all_file_action and add commentaries
-# Available actions:
-# keep - keep license in contrib and use in credits
-# skip - skip license
-# remove - remove all files with this license
-# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file
-#
-# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory)
-# We suppose that that files can contain some license info
-# Available all file actions:
-# FILE_IGNORE - ignore file (do nothing)
-# FILE_INCLUDE - include all file data into licenses text file
-# =======================
-
-KEEP COPYRIGHT_SERVICE_LABEL 4d1693dc5c86412be2389965e0fe6e43
-BELONGS ya.make
- License text:
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- lzo_crc.c [5:6]
- lzoconf.h [5:6]
- lzodefs.h [5:6]
- minilzo.c [5:6]
- minilzo.h [5:6]
-
-KEEP COPYRIGHT_SERVICE_LABEL 69624c87e8384824adcf31e753c4552d
-BELONGS ya.make
- License text:
- "$Copyright: LZO Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer\n"
- "<markus@oberhumer.com>\n"
- "http://www.oberhumer.com $\n\n"
- Scancode info:
- Original SPDX id: COPYRIGHT_SERVICE_LABEL
- Score : 100.00
- Match type : COPYRIGHT
- Files with this license:
- minilzo.c [4094:4096]
diff --git a/contrib/libs/minilzo/.yandex_meta/devtools.licenses.report b/contrib/libs/minilzo/.yandex_meta/devtools.licenses.report
deleted file mode 100644
index 0ffa0489b9..0000000000
--- a/contrib/libs/minilzo/.yandex_meta/devtools.licenses.report
+++ /dev/null
@@ -1,45 +0,0 @@
-# File format ($ symbol means the beginning of a line):
-#
-# $ # this message
-# $ # =======================
-# $ # comments (all commentaries should starts with some number of spaces and # symbol)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make
-# ${all_file_action} filename
-# $ # user commentaries (many lines)
-# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify)
-# ${action} {license spdx} {license text hash}
-# $BELONGS ./ya/make/file/relative/path/3/ya.make
-# ${all_file_action} filename
-# $ # user commentaries
-# $ generated description
-# $ ...
-#
-# You can modify action, all_file_action and add commentaries
-# Available actions:
-# keep - keep license in contrib and use in credits
-# skip - skip license
-# remove - remove all files with this license
-# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file
-#
-# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory)
-# We suppose that that files can contain some license info
-# Available all file actions:
-# FILE_IGNORE - ignore file (do nothing)
-# FILE_INCLUDE - include all file data into licenses text file
-# =======================
-
-KEEP GPL-2.0-or-later d5af58ff34570116fd26549300287c61
-BELONGS ya.make
- Note: matched license text is too long. Read it in the source files.
- Scancode info:
- Original SPDX id: GPL-2.0-or-later
- Score : 100.00
- Match type : NOTICE
- Links : http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html, https://spdx.org/licenses/GPL-2.0-or-later
- Files with this license:
- lzo_crc.c [8:21]
- lzoconf.h [8:21]
- lzodefs.h [8:21]
- minilzo.c [8:21]
- minilzo.h [8:21]
diff --git a/contrib/libs/minilzo/.yandex_meta/licenses.list.txt b/contrib/libs/minilzo/.yandex_meta/licenses.list.txt
deleted file mode 100644
index 324bb9b9c3..0000000000
--- a/contrib/libs/minilzo/.yandex_meta/licenses.list.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-====================COPYRIGHT====================
- "$Copyright: LZO Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer\n"
- "<markus@oberhumer.com>\n"
- "http://www.oberhumer.com $\n\n"
-
-
-====================COPYRIGHT====================
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
-
-====================GPL-2.0-or-later====================
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
diff --git a/contrib/libs/minilzo/CMakeLists.txt b/contrib/libs/minilzo/CMakeLists.txt
deleted file mode 100644
index 4ce71359eb..0000000000
--- a/contrib/libs/minilzo/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(contrib-libs-minilzo)
-target_compile_options(contrib-libs-minilzo PRIVATE
- -DMINILZO_CFG_SKIP_LZO_STRING
-)
-target_link_libraries(contrib-libs-minilzo PUBLIC
- contrib-libs-cxxsupp
-)
-target_sources(contrib-libs-minilzo PRIVATE
- ${CMAKE_SOURCE_DIR}/contrib/libs/minilzo/minilzo.c
- ${CMAKE_SOURCE_DIR}/contrib/libs/minilzo/lzo_crc.c
-)
diff --git a/contrib/libs/minilzo/lzo_crc.c b/contrib/libs/minilzo/lzo_crc.c
deleted file mode 100644
index 19e40239df..0000000000
--- a/contrib/libs/minilzo/lzo_crc.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/* lzo_crc.c -- crc checksum for the the LZO library
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
- Markus F.X.J. Oberhumer
- <markus@oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-
-#include "protect.h"
-#include "lzoconf.h"
-
-
-/***********************************************************************
-// crc32 checksum
-// adapted from free code by Mark Adler <madler@alumni.caltech.edu>
-// see http://www.zlib.org/
-************************************************************************/
-
-static const lzo_uint32_t lzo_crc32_table[256] = {
- 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
- 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
- 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
- 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
- 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
- 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
- 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
- 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
- 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
- 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
- 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
- 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
- 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
- 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
- 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
- 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
- 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
- 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
- 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
- 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
- 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
- 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
- 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
- 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
- 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
- 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
- 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
- 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
- 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
- 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
- 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
- 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
- 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
- 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
- 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
- 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
- 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
- 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
- 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
- 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
- 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
- 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
- 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
- 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
- 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
- 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
- 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
- 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
- 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
- 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
- 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
- 0x2d02ef8dL
-};
-
-
-LZO_PUBLIC(const lzo_uint32_tp)
-lzo_get_crc32_table(void)
-{
- return lzo_crc32_table;
-}
-
-
-#if 1
-#define LZO_DO1(buf,i) \
- crc = table[((unsigned)crc ^ buf[i]) & 0xff] ^ (crc >> 8)
-#else
-#define LZO_DO1(buf,i) \
- crc = table[(unsigned char)((unsigned char)crc ^ buf[i])] ^ (crc >> 8)
-#endif
-#define LZO_DO2(buf,i) LZO_DO1(buf,i); LZO_DO1(buf,i+1)
-#define LZO_DO4(buf,i) LZO_DO2(buf,i); LZO_DO2(buf,i+2)
-#define LZO_DO8(buf,i) LZO_DO4(buf,i); LZO_DO4(buf,i+4)
-#define LZO_DO16(buf,i) LZO_DO8(buf,i); LZO_DO8(buf,i+8)
-
-
-LZO_PUBLIC(lzo_uint32_t)
-lzo_crc32(lzo_uint32_t c, const lzo_bytep buf, lzo_uint len)
-{
- lzo_uint32_t crc;
-#undef table
-#if 1
-# define table lzo_crc32_table
-#else
- const lzo_uint32_t * table = lzo_crc32_table;
-#endif
-
- if (buf == NULL)
- return 0;
-
- crc = (c & LZO_UINT32_C(0xffffffff)) ^ LZO_UINT32_C(0xffffffff);
- if (len >= 16) do
- {
- LZO_DO16(buf,0);
- buf += 16;
- len -= 16;
- } while (len >= 16);
- if (len != 0) do
- {
- LZO_DO1(buf,0);
- buf += 1;
- len -= 1;
- } while (len > 0);
-
- return crc ^ LZO_UINT32_C(0xffffffff);
-#undef table
-}
-
-#undef LZO_DO1
-#undef LZO_DO2
-#undef LZO_DO4
-#undef LZO_DO8
-#undef LZO_DO16
-
-
-/* vim:set ts=4 sw=4 et: */
diff --git a/contrib/libs/minilzo/lzoconf.h b/contrib/libs/minilzo/lzoconf.h
deleted file mode 100644
index c713c4291e..0000000000
--- a/contrib/libs/minilzo/lzoconf.h
+++ /dev/null
@@ -1,453 +0,0 @@
-/* lzoconf.h -- configuration of the LZO data compression library
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
- Markus F.X.J. Oberhumer
- <markus@oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-
-#ifndef __LZOCONF_H_INCLUDED
-#define __LZOCONF_H_INCLUDED 1
-
-#define LZO_VERSION 0x2090
-#define LZO_VERSION_STRING "2.09"
-#define LZO_VERSION_DATE "Feb 04 2015"
-
-/* internal Autoconf configuration file - only used when building LZO */
-#if defined(LZO_HAVE_CONFIG_H)
-#error # include <config.h>
-#endif
-#include <limits.h>
-#include <stddef.h>
-
-
-/***********************************************************************
-// LZO requires a conforming <limits.h>
-************************************************************************/
-
-#if !defined(CHAR_BIT) || (CHAR_BIT != 8)
-# error "invalid CHAR_BIT"
-#endif
-#if !defined(UCHAR_MAX) || !defined(USHRT_MAX) || !defined(UINT_MAX) || !defined(ULONG_MAX)
-# error "check your compiler installation"
-#endif
-#if (USHRT_MAX < 1) || (UINT_MAX < 1) || (ULONG_MAX < 1)
-# error "your limits.h macros are broken"
-#endif
-
-/* get OS and architecture defines */
-#ifndef __LZODEFS_H_INCLUDED
-#include "lzodefs.h"
-#endif
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/***********************************************************************
-// some core defines
-************************************************************************/
-
-/* memory checkers */
-#if !defined(__LZO_CHECKER)
-# if defined(__BOUNDS_CHECKING_ON)
-# define __LZO_CHECKER 1
-# elif defined(__CHECKER__)
-# define __LZO_CHECKER 1
-# elif defined(__INSURE__)
-# define __LZO_CHECKER 1
-# elif defined(__PURIFY__)
-# define __LZO_CHECKER 1
-# endif
-#endif
-
-
-/***********************************************************************
-// integral and pointer types
-************************************************************************/
-
-/* lzo_uint must match size_t */
-#if !defined(LZO_UINT_MAX)
-# if (LZO_ABI_LLP64)
-# if (LZO_OS_WIN64)
- typedef unsigned __int64 lzo_uint;
- typedef __int64 lzo_int;
-# define LZO_TYPEOF_LZO_INT LZO_TYPEOF___INT64
-# else
- typedef lzo_ullong_t lzo_uint;
- typedef lzo_llong_t lzo_int;
-# define LZO_TYPEOF_LZO_INT LZO_TYPEOF_LONG_LONG
-# endif
-# define LZO_SIZEOF_LZO_INT 8
-# define LZO_UINT_MAX 0xffffffffffffffffull
-# define LZO_INT_MAX 9223372036854775807LL
-# define LZO_INT_MIN (-1LL - LZO_INT_MAX)
-# elif (LZO_ABI_IP32L64) /* MIPS R5900 */
- typedef unsigned int lzo_uint;
- typedef int lzo_int;
-# define LZO_SIZEOF_LZO_INT LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INT LZO_TYPEOF_INT
-# define LZO_UINT_MAX UINT_MAX
-# define LZO_INT_MAX INT_MAX
-# define LZO_INT_MIN INT_MIN
-# elif (ULONG_MAX >= LZO_0xffffffffL)
- typedef unsigned long lzo_uint;
- typedef long lzo_int;
-# define LZO_SIZEOF_LZO_INT LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_INT LZO_TYPEOF_LONG
-# define LZO_UINT_MAX ULONG_MAX
-# define LZO_INT_MAX LONG_MAX
-# define LZO_INT_MIN LONG_MIN
-# else
-# error "lzo_uint"
-# endif
-#endif
-
-/* The larger type of lzo_uint and lzo_uint32_t. */
-#if (LZO_SIZEOF_LZO_INT >= 4)
-# define lzo_xint lzo_uint
-#else
-# define lzo_xint lzo_uint32_t
-#endif
-
-typedef int lzo_bool;
-
-/* sanity checks */
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int) == LZO_SIZEOF_LZO_INT)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint) == LZO_SIZEOF_LZO_INT)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_xint) >= sizeof(lzo_uint))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_xint) >= sizeof(lzo_uint32_t))
-
-#ifndef __LZO_MMODEL
-#define __LZO_MMODEL /*empty*/
-#endif
-
-/* no typedef here because of const-pointer issues */
-#define lzo_bytep unsigned char __LZO_MMODEL *
-#define lzo_charp char __LZO_MMODEL *
-#define lzo_voidp void __LZO_MMODEL *
-#define lzo_shortp short __LZO_MMODEL *
-#define lzo_ushortp unsigned short __LZO_MMODEL *
-#define lzo_intp lzo_int __LZO_MMODEL *
-#define lzo_uintp lzo_uint __LZO_MMODEL *
-#define lzo_xintp lzo_xint __LZO_MMODEL *
-#define lzo_voidpp lzo_voidp __LZO_MMODEL *
-#define lzo_bytepp lzo_bytep __LZO_MMODEL *
-
-#define lzo_int8_tp lzo_int8_t __LZO_MMODEL *
-#define lzo_uint8_tp lzo_uint8_t __LZO_MMODEL *
-#define lzo_int16_tp lzo_int16_t __LZO_MMODEL *
-#define lzo_uint16_tp lzo_uint16_t __LZO_MMODEL *
-#define lzo_int32_tp lzo_int32_t __LZO_MMODEL *
-#define lzo_uint32_tp lzo_uint32_t __LZO_MMODEL *
-#if defined(lzo_int64_t)
-#define lzo_int64_tp lzo_int64_t __LZO_MMODEL *
-#define lzo_uint64_tp lzo_uint64_t __LZO_MMODEL *
-#endif
-
-/* Older LZO versions used to support ancient systems and memory models
- * such as 16-bit MSDOS with __huge pointers or Cray PVP, but these
- * obsolete configurations are not supported any longer.
- */
-#if defined(__LZO_MMODEL_HUGE)
-#error "__LZO_MMODEL_HUGE memory model is unsupported"
-#endif
-#if (LZO_MM_PVP)
-#error "LZO_MM_PVP memory model is unsupported"
-#endif
-#if (LZO_SIZEOF_INT < 4)
-#error "LZO_SIZEOF_INT < 4 is unsupported"
-#endif
-#if (__LZO_UINTPTR_T_IS_POINTER)
-#error "__LZO_UINTPTR_T_IS_POINTER is unsupported"
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(int) >= 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint) >= 4)
-/* Strange configurations where sizeof(lzo_uint) != sizeof(size_t) should
- * work but have not received much testing lately, so be strict here.
- */
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint) == sizeof(size_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint) == sizeof(ptrdiff_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint) == sizeof(lzo_uintptr_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(void *) == sizeof(lzo_uintptr_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(char *) == sizeof(lzo_uintptr_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long *) == sizeof(lzo_uintptr_t))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(void *) == sizeof(lzo_voidp))
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(char *) == sizeof(lzo_bytep))
-
-
-/***********************************************************************
-// function types
-************************************************************************/
-
-/* name mangling */
-#if !defined(__LZO_EXTERN_C)
-# ifdef __cplusplus
-# define __LZO_EXTERN_C extern "C"
-# else
-# define __LZO_EXTERN_C extern
-# endif
-#endif
-
-/* calling convention */
-#if !defined(__LZO_CDECL)
-# define __LZO_CDECL __lzo_cdecl
-#endif
-
-/* DLL export information */
-#if !defined(__LZO_EXPORT1)
-# define __LZO_EXPORT1 /*empty*/
-#endif
-#if !defined(__LZO_EXPORT2)
-# define __LZO_EXPORT2 /*empty*/
-#endif
-
-/* __cdecl calling convention for public C and assembly functions */
-#if !defined(LZO_PUBLIC)
-# define LZO_PUBLIC(r) __LZO_EXPORT1 r __LZO_EXPORT2 __LZO_CDECL
-#endif
-#if !defined(LZO_EXTERN)
-# define LZO_EXTERN(r) __LZO_EXTERN_C LZO_PUBLIC(r)
-#endif
-#if !defined(LZO_PRIVATE)
-# define LZO_PRIVATE(r) static r __LZO_CDECL
-#endif
-
-/* function types */
-typedef int
-(__LZO_CDECL *lzo_compress_t) ( const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem );
-
-typedef int
-(__LZO_CDECL *lzo_decompress_t) ( const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem );
-
-typedef int
-(__LZO_CDECL *lzo_optimize_t) ( lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem );
-
-typedef int
-(__LZO_CDECL *lzo_compress_dict_t)(const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem,
- const lzo_bytep dict, lzo_uint dict_len );
-
-typedef int
-(__LZO_CDECL *lzo_decompress_dict_t)(const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem,
- const lzo_bytep dict, lzo_uint dict_len );
-
-
-/* Callback interface. Currently only the progress indicator ("nprogress")
- * is used, but this may change in a future release. */
-
-struct lzo_callback_t;
-typedef struct lzo_callback_t lzo_callback_t;
-#define lzo_callback_p lzo_callback_t __LZO_MMODEL *
-
-/* malloc & free function types */
-typedef lzo_voidp (__LZO_CDECL *lzo_alloc_func_t)
- (lzo_callback_p self, lzo_uint items, lzo_uint size);
-typedef void (__LZO_CDECL *lzo_free_func_t)
- (lzo_callback_p self, lzo_voidp ptr);
-
-/* a progress indicator callback function */
-typedef void (__LZO_CDECL *lzo_progress_func_t)
- (lzo_callback_p, lzo_uint, lzo_uint, int);
-
-struct lzo_callback_t
-{
- /* custom allocators (set to 0 to disable) */
- lzo_alloc_func_t nalloc; /* [not used right now] */
- lzo_free_func_t nfree; /* [not used right now] */
-
- /* a progress indicator callback function (set to 0 to disable) */
- lzo_progress_func_t nprogress;
-
- /* INFO: the first parameter "self" of the nalloc/nfree/nprogress
- * callbacks points back to this struct, so you are free to store
- * some extra info in the following variables. */
- lzo_voidp user1;
- lzo_xint user2;
- lzo_xint user3;
-};
-
-
-/***********************************************************************
-// error codes and prototypes
-************************************************************************/
-
-/* Error codes for the compression/decompression functions. Negative
- * values are errors, positive values will be used for special but
- * normal events.
- */
-#define LZO_E_OK 0
-#define LZO_E_ERROR (-1)
-#define LZO_E_OUT_OF_MEMORY (-2) /* [lzo_alloc_func_t failure] */
-#define LZO_E_NOT_COMPRESSIBLE (-3) /* [not used right now] */
-#define LZO_E_INPUT_OVERRUN (-4)
-#define LZO_E_OUTPUT_OVERRUN (-5)
-#define LZO_E_LOOKBEHIND_OVERRUN (-6)
-#define LZO_E_EOF_NOT_FOUND (-7)
-#define LZO_E_INPUT_NOT_CONSUMED (-8)
-#define LZO_E_NOT_YET_IMPLEMENTED (-9) /* [not used right now] */
-#define LZO_E_INVALID_ARGUMENT (-10)
-#define LZO_E_INVALID_ALIGNMENT (-11) /* pointer argument is not properly aligned */
-#define LZO_E_OUTPUT_NOT_CONSUMED (-12)
-#define LZO_E_INTERNAL_ERROR (-99)
-
-
-#ifndef lzo_sizeof_dict_t
-# define lzo_sizeof_dict_t ((unsigned)sizeof(lzo_bytep))
-#endif
-
-/* lzo_init() should be the first function you call.
- * Check the return code !
- *
- * lzo_init() is a macro to allow checking that the library and the
- * compiler's view of various types are consistent.
- */
-#define lzo_init() __lzo_init_v2(LZO_VERSION,(int)sizeof(short),(int)sizeof(int),\
- (int)sizeof(long),(int)sizeof(lzo_uint32_t),(int)sizeof(lzo_uint),\
- (int)lzo_sizeof_dict_t,(int)sizeof(char *),(int)sizeof(lzo_voidp),\
- (int)sizeof(lzo_callback_t))
-LZO_EXTERN(int) __lzo_init_v2(unsigned,int,int,int,int,int,int,int,int,int);
-
-/* version functions (useful for shared libraries) */
-LZO_EXTERN(unsigned) lzo_version(void);
-LZO_EXTERN(const char *) lzo_version_string(void);
-LZO_EXTERN(const char *) lzo_version_date(void);
-LZO_EXTERN(const lzo_charp) _lzo_version_string(void);
-LZO_EXTERN(const lzo_charp) _lzo_version_date(void);
-
-/* string functions */
-LZO_EXTERN(int)
- lzo_memcmp(const lzo_voidp a, const lzo_voidp b, lzo_uint len);
-LZO_EXTERN(lzo_voidp)
- lzo_memcpy(lzo_voidp dst, const lzo_voidp src, lzo_uint len);
-LZO_EXTERN(lzo_voidp)
- lzo_memmove(lzo_voidp dst, const lzo_voidp src, lzo_uint len);
-LZO_EXTERN(lzo_voidp)
- lzo_memset(lzo_voidp buf, int c, lzo_uint len);
-
-/* checksum functions */
-LZO_EXTERN(lzo_uint32_t)
- lzo_adler32(lzo_uint32_t c, const lzo_bytep buf, lzo_uint len);
-LZO_EXTERN(lzo_uint32_t)
- lzo_crc32(lzo_uint32_t c, const lzo_bytep buf, lzo_uint len);
-LZO_EXTERN(const lzo_uint32_tp)
- lzo_get_crc32_table(void);
-
-/* misc. */
-LZO_EXTERN(int) _lzo_config_check(void);
-typedef union {
- lzo_voidp a00; lzo_bytep a01; lzo_uint a02; lzo_xint a03; lzo_uintptr_t a04;
- void *a05; unsigned char *a06; unsigned long a07; size_t a08; ptrdiff_t a09;
-#if defined(lzo_int64_t)
- lzo_uint64_t a10;
-#endif
-} lzo_align_t;
-
-/* align a char pointer on a boundary that is a multiple of 'size' */
-LZO_EXTERN(unsigned) __lzo_align_gap(const lzo_voidp p, lzo_uint size);
-#define LZO_PTR_ALIGN_UP(p,size) \
- ((p) + (lzo_uint) __lzo_align_gap((const lzo_voidp)(p),(lzo_uint)(size)))
-
-
-/***********************************************************************
-// deprecated macros - only for backward compatibility
-************************************************************************/
-
-/* deprecated - use 'lzo_bytep' instead of 'lzo_byte *' */
-#define lzo_byte unsigned char
-/* deprecated type names */
-#define lzo_int32 lzo_int32_t
-#define lzo_uint32 lzo_uint32_t
-#define lzo_int32p lzo_int32_t __LZO_MMODEL *
-#define lzo_uint32p lzo_uint32_t __LZO_MMODEL *
-#define LZO_INT32_MAX LZO_INT32_C(2147483647)
-#define LZO_UINT32_MAX LZO_UINT32_C(4294967295)
-#if defined(lzo_int64_t)
-#define lzo_int64 lzo_int64_t
-#define lzo_uint64 lzo_uint64_t
-#define lzo_int64p lzo_int64_t __LZO_MMODEL *
-#define lzo_uint64p lzo_uint64_t __LZO_MMODEL *
-#define LZO_INT64_MAX LZO_INT64_C(9223372036854775807)
-#define LZO_UINT64_MAX LZO_UINT64_C(18446744073709551615)
-#endif
-/* deprecated types */
-typedef union { lzo_bytep a; lzo_uint b; } __lzo_pu_u;
-typedef union { lzo_bytep a; lzo_uint32_t b; } __lzo_pu32_u;
-/* deprecated defines */
-#if !defined(LZO_SIZEOF_LZO_UINT)
-# define LZO_SIZEOF_LZO_UINT LZO_SIZEOF_LZO_INT
-#endif
-
-#if defined(LZO_CFG_COMPAT)
-
-#define __LZOCONF_H 1
-
-#if defined(LZO_ARCH_I086)
-# define __LZO_i386 1
-#elif defined(LZO_ARCH_I386)
-# define __LZO_i386 1
-#endif
-
-#if defined(LZO_OS_DOS16)
-# define __LZO_DOS 1
-# define __LZO_DOS16 1
-#elif defined(LZO_OS_DOS32)
-# define __LZO_DOS 1
-#elif defined(LZO_OS_WIN16)
-# define __LZO_WIN 1
-# define __LZO_WIN16 1
-#elif defined(LZO_OS_WIN32)
-# define __LZO_WIN 1
-#endif
-
-#define __LZO_CMODEL /*empty*/
-#define __LZO_DMODEL /*empty*/
-#define __LZO_ENTRY __LZO_CDECL
-#define LZO_EXTERN_CDECL LZO_EXTERN
-#define LZO_ALIGN LZO_PTR_ALIGN_UP
-
-#define lzo_compress_asm_t lzo_compress_t
-#define lzo_decompress_asm_t lzo_decompress_t
-
-#endif /* LZO_CFG_COMPAT */
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* already included */
-
-
-/* vim:set ts=4 sw=4 et: */
diff --git a/contrib/libs/minilzo/lzodefs.h b/contrib/libs/minilzo/lzodefs.h
deleted file mode 100644
index 1535c1e205..0000000000
--- a/contrib/libs/minilzo/lzodefs.h
+++ /dev/null
@@ -1,3134 +0,0 @@
-/* lzodefs.h -- architecture, OS and compiler specific defines
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
- Markus F.X.J. Oberhumer
- <markus@oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-
-#ifndef __LZODEFS_H_INCLUDED
-#define __LZODEFS_H_INCLUDED 1
-
-#if defined(__CYGWIN32__) && !defined(__CYGWIN__)
-# define __CYGWIN__ __CYGWIN32__
-#endif
-#if 1 && defined(__INTERIX) && defined(__GNUC__) && !defined(_ALL_SOURCE)
-# define _ALL_SOURCE 1
-#endif
-#if defined(__mips__) && defined(__R5900__)
-# if !defined(__LONG_MAX__)
-# define __LONG_MAX__ 9223372036854775807L
-# endif
-#endif
-#if !defined(LZO_CFG_NO_DISABLE_WUNDEF)
-#if defined(__ARMCC_VERSION)
-# pragma diag_suppress 193
-#elif defined(__clang__) && defined(__clang_minor__)
-# pragma clang diagnostic ignored "-Wundef"
-#elif defined(__INTEL_COMPILER)
-# pragma warning(disable: 193)
-#elif defined(__KEIL__) && defined(__C166__)
-# pragma warning disable = 322
-#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && !defined(__PATHSCALE__)
-# if ((__GNUC__-0) >= 5 || ((__GNUC__-0) == 4 && (__GNUC_MINOR__-0) >= 2))
-# pragma GCC diagnostic ignored "-Wundef"
-# endif
-#elif defined(_MSC_VER) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__MWERKS__)
-# if ((_MSC_VER-0) >= 1300)
-# pragma warning(disable: 4668)
-# endif
-#endif
-#endif
-#if 0 && defined(__POCC__) && defined(_WIN32)
-# if (__POCC__ >= 400)
-# pragma warn(disable: 2216)
-# endif
-#endif
-#if 0 && defined(__WATCOMC__)
-# if (__WATCOMC__ >= 1050) && (__WATCOMC__ < 1060)
-# pragma warning 203 9
-# endif
-#endif
-#if defined(__BORLANDC__) && defined(__MSDOS__) && !defined(__FLAT__)
-# pragma option -h
-#endif
-#if !(LZO_CFG_NO_DISABLE_WCRTNONSTDC)
-#ifndef _CRT_NONSTDC_NO_DEPRECATE
-#define _CRT_NONSTDC_NO_DEPRECATE 1
-#endif
-#ifndef _CRT_NONSTDC_NO_WARNINGS
-#define _CRT_NONSTDC_NO_WARNINGS 1
-#endif
-#ifndef _CRT_SECURE_NO_DEPRECATE
-#define _CRT_SECURE_NO_DEPRECATE 1
-#endif
-#ifndef _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_WARNINGS 1
-#endif
-#endif
-#if 0
-#define LZO_0xffffUL 0xfffful
-#define LZO_0xffffffffUL 0xfffffffful
-#else
-#define LZO_0xffffUL 65535ul
-#define LZO_0xffffffffUL 4294967295ul
-#endif
-#define LZO_0xffffL LZO_0xffffUL
-#define LZO_0xffffffffL LZO_0xffffffffUL
-#if (LZO_0xffffL == LZO_0xffffffffL)
-# error "your preprocessor is broken 1"
-#endif
-#if (16ul * 16384ul != 262144ul)
-# error "your preprocessor is broken 2"
-#endif
-#if 0
-#if (32767 >= 4294967295ul)
-# error "your preprocessor is broken 3"
-#endif
-#if (65535u >= 4294967295ul)
-# error "your preprocessor is broken 4"
-#endif
-#endif
-#if defined(__COUNTER__)
-# ifndef LZO_CFG_USE_COUNTER
-# define LZO_CFG_USE_COUNTER 1
-# endif
-#else
-# undef LZO_CFG_USE_COUNTER
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-#if defined(__ZTC__) && defined(__I86__) && !defined(__OS2__)
-# if !defined(MSDOS)
-# define MSDOS 1
-# endif
-# if !defined(_MSDOS)
-# define _MSDOS 1
-# endif
-#elif 0 && defined(__VERSION) && defined(MB_LEN_MAX)
-# if (__VERSION == 520) && (MB_LEN_MAX == 1)
-# if !defined(__AZTEC_C__)
-# define __AZTEC_C__ __VERSION
-# endif
-# if !defined(__DOS__)
-# define __DOS__ 1
-# endif
-# endif
-#endif
-#endif
-#if defined(_MSC_VER) && defined(M_I86HM) && (UINT_MAX == LZO_0xffffL)
-# define ptrdiff_t long
-# define _PTRDIFF_T_DEFINED 1
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-# undef __LZO_RENAME_A
-# undef __LZO_RENAME_B
-# if defined(__AZTEC_C__) && defined(__DOS__)
-# define __LZO_RENAME_A 1
-# elif defined(_MSC_VER) && defined(MSDOS)
-# if (_MSC_VER < 600)
-# define __LZO_RENAME_A 1
-# elif (_MSC_VER < 700)
-# define __LZO_RENAME_B 1
-# endif
-# elif defined(__TSC__) && defined(__OS2__)
-# define __LZO_RENAME_A 1
-# elif defined(__MSDOS__) && defined(__TURBOC__) && (__TURBOC__ < 0x0410)
-# define __LZO_RENAME_A 1
-# elif defined(__PACIFIC__) && defined(DOS)
-# if !defined(__far)
-# define __far far
-# endif
-# if !defined(__near)
-# define __near near
-# endif
-# endif
-# if defined(__LZO_RENAME_A)
-# if !defined(__cdecl)
-# define __cdecl cdecl
-# endif
-# if !defined(__far)
-# define __far far
-# endif
-# if !defined(__huge)
-# define __huge huge
-# endif
-# if !defined(__near)
-# define __near near
-# endif
-# if !defined(__pascal)
-# define __pascal pascal
-# endif
-# if !defined(__huge)
-# define __huge huge
-# endif
-# elif defined(__LZO_RENAME_B)
-# if !defined(__cdecl)
-# define __cdecl _cdecl
-# endif
-# if !defined(__far)
-# define __far _far
-# endif
-# if !defined(__huge)
-# define __huge _huge
-# endif
-# if !defined(__near)
-# define __near _near
-# endif
-# if !defined(__pascal)
-# define __pascal _pascal
-# endif
-# elif (defined(__PUREC__) || defined(__TURBOC__)) && defined(__TOS__)
-# if !defined(__cdecl)
-# define __cdecl cdecl
-# endif
-# if !defined(__pascal)
-# define __pascal pascal
-# endif
-# endif
-# undef __LZO_RENAME_A
-# undef __LZO_RENAME_B
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-#if defined(__AZTEC_C__) && defined(__DOS__)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-#elif defined(_MSC_VER) && defined(MSDOS)
-# if (_MSC_VER < 600)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-# endif
-# if (_MSC_VER < 700)
-# define LZO_BROKEN_INTEGRAL_PROMOTION 1
-# define LZO_BROKEN_SIZEOF 1
-# endif
-#elif defined(__PACIFIC__) && defined(DOS)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-#elif defined(__TURBOC__) && defined(__MSDOS__)
-# if (__TURBOC__ < 0x0150)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-# define LZO_BROKEN_INTEGRAL_PROMOTION 1
-# endif
-# if (__TURBOC__ < 0x0200)
-# define LZO_BROKEN_SIZEOF 1
-# endif
-# if (__TURBOC__ < 0x0400) && defined(__cplusplus)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# endif
-#elif (defined(__PUREC__) || defined(__TURBOC__)) && defined(__TOS__)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# define LZO_BROKEN_SIZEOF 1
-#endif
-#endif
-#if defined(__WATCOMC__) && (__WATCOMC__ < 900)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-#endif
-#if defined(_CRAY) && defined(_CRAY1)
-# define LZO_BROKEN_SIGNED_RIGHT_SHIFT 1
-#endif
-#define LZO_PP_STRINGIZE(x) #x
-#define LZO_PP_MACRO_EXPAND(x) LZO_PP_STRINGIZE(x)
-#define LZO_PP_CONCAT0() /*empty*/
-#define LZO_PP_CONCAT1(a) a
-#define LZO_PP_CONCAT2(a,b) a ## b
-#define LZO_PP_CONCAT3(a,b,c) a ## b ## c
-#define LZO_PP_CONCAT4(a,b,c,d) a ## b ## c ## d
-#define LZO_PP_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e
-#define LZO_PP_CONCAT6(a,b,c,d,e,f) a ## b ## c ## d ## e ## f
-#define LZO_PP_CONCAT7(a,b,c,d,e,f,g) a ## b ## c ## d ## e ## f ## g
-#define LZO_PP_ECONCAT0() LZO_PP_CONCAT0()
-#define LZO_PP_ECONCAT1(a) LZO_PP_CONCAT1(a)
-#define LZO_PP_ECONCAT2(a,b) LZO_PP_CONCAT2(a,b)
-#define LZO_PP_ECONCAT3(a,b,c) LZO_PP_CONCAT3(a,b,c)
-#define LZO_PP_ECONCAT4(a,b,c,d) LZO_PP_CONCAT4(a,b,c,d)
-#define LZO_PP_ECONCAT5(a,b,c,d,e) LZO_PP_CONCAT5(a,b,c,d,e)
-#define LZO_PP_ECONCAT6(a,b,c,d,e,f) LZO_PP_CONCAT6(a,b,c,d,e,f)
-#define LZO_PP_ECONCAT7(a,b,c,d,e,f,g) LZO_PP_CONCAT7(a,b,c,d,e,f,g)
-#define LZO_PP_EMPTY /*empty*/
-#define LZO_PP_EMPTY0() /*empty*/
-#define LZO_PP_EMPTY1(a) /*empty*/
-#define LZO_PP_EMPTY2(a,b) /*empty*/
-#define LZO_PP_EMPTY3(a,b,c) /*empty*/
-#define LZO_PP_EMPTY4(a,b,c,d) /*empty*/
-#define LZO_PP_EMPTY5(a,b,c,d,e) /*empty*/
-#define LZO_PP_EMPTY6(a,b,c,d,e,f) /*empty*/
-#define LZO_PP_EMPTY7(a,b,c,d,e,f,g) /*empty*/
-#if 1
-#define LZO_CPP_STRINGIZE(x) #x
-#define LZO_CPP_MACRO_EXPAND(x) LZO_CPP_STRINGIZE(x)
-#define LZO_CPP_CONCAT2(a,b) a ## b
-#define LZO_CPP_CONCAT3(a,b,c) a ## b ## c
-#define LZO_CPP_CONCAT4(a,b,c,d) a ## b ## c ## d
-#define LZO_CPP_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e
-#define LZO_CPP_CONCAT6(a,b,c,d,e,f) a ## b ## c ## d ## e ## f
-#define LZO_CPP_CONCAT7(a,b,c,d,e,f,g) a ## b ## c ## d ## e ## f ## g
-#define LZO_CPP_ECONCAT2(a,b) LZO_CPP_CONCAT2(a,b)
-#define LZO_CPP_ECONCAT3(a,b,c) LZO_CPP_CONCAT3(a,b,c)
-#define LZO_CPP_ECONCAT4(a,b,c,d) LZO_CPP_CONCAT4(a,b,c,d)
-#define LZO_CPP_ECONCAT5(a,b,c,d,e) LZO_CPP_CONCAT5(a,b,c,d,e)
-#define LZO_CPP_ECONCAT6(a,b,c,d,e,f) LZO_CPP_CONCAT6(a,b,c,d,e,f)
-#define LZO_CPP_ECONCAT7(a,b,c,d,e,f,g) LZO_CPP_CONCAT7(a,b,c,d,e,f,g)
-#endif
-#define __LZO_MASK_GEN(o,b) (((((o) << ((b)-!!(b))) - (o)) << 1) + (o)*!!(b))
-#if 1 && defined(__cplusplus)
-# if !defined(__STDC_CONSTANT_MACROS)
-# define __STDC_CONSTANT_MACROS 1
-# endif
-# if !defined(__STDC_LIMIT_MACROS)
-# define __STDC_LIMIT_MACROS 1
-# endif
-#endif
-#if defined(__cplusplus)
-# define LZO_EXTERN_C extern "C"
-# define LZO_EXTERN_C_BEGIN extern "C" {
-# define LZO_EXTERN_C_END }
-#else
-# define LZO_EXTERN_C extern
-# define LZO_EXTERN_C_BEGIN /*empty*/
-# define LZO_EXTERN_C_END /*empty*/
-#endif
-#if !defined(__LZO_OS_OVERRIDE)
-#if (LZO_OS_FREESTANDING)
-# define LZO_INFO_OS "freestanding"
-#elif (LZO_OS_EMBEDDED)
-# define LZO_INFO_OS "embedded"
-#elif 1 && defined(__IAR_SYSTEMS_ICC__)
-# define LZO_OS_EMBEDDED 1
-# define LZO_INFO_OS "embedded"
-#elif defined(__CYGWIN__) && defined(__GNUC__)
-# define LZO_OS_CYGWIN 1
-# define LZO_INFO_OS "cygwin"
-#elif defined(__EMX__) && defined(__GNUC__)
-# define LZO_OS_EMX 1
-# define LZO_INFO_OS "emx"
-#elif defined(__BEOS__)
-# define LZO_OS_BEOS 1
-# define LZO_INFO_OS "beos"
-#elif defined(__Lynx__)
-# define LZO_OS_LYNXOS 1
-# define LZO_INFO_OS "lynxos"
-#elif defined(__OS400__)
-# define LZO_OS_OS400 1
-# define LZO_INFO_OS "os400"
-#elif defined(__QNX__)
-# define LZO_OS_QNX 1
-# define LZO_INFO_OS "qnx"
-#elif defined(__BORLANDC__) && defined(__DPMI32__) && (__BORLANDC__ >= 0x0460)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-#elif defined(__BORLANDC__) && defined(__DPMI16__)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-#elif defined(__ZTC__) && defined(DOS386)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-#elif defined(__OS2__) || defined(__OS2V2__)
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_OS216 1
-# define LZO_INFO_OS "os216"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_OS2 1
-# define LZO_INFO_OS "os2"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__WIN64__) || defined(_WIN64) || defined(WIN64)
-# define LZO_OS_WIN64 1
-# define LZO_INFO_OS "win64"
-#elif defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(__WINDOWS_386__)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-#elif defined(__MWERKS__) && defined(__INTEL__)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-#elif defined(__WINDOWS__) || defined(_WINDOWS) || defined(_Windows)
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_WIN16 1
-# define LZO_INFO_OS "win16"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__DOS__) || defined(__MSDOS__) || defined(_MSDOS) || defined(MSDOS) || (defined(__PACIFIC__) && defined(DOS))
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__WATCOMC__)
-# if defined(__NT__) && (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-# elif defined(__NT__) && (__WATCOMC__ < 1100)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-# elif defined(__linux__) || defined(__LINUX__)
-# define LZO_OS_POSIX 1
-# define LZO_INFO_OS "posix"
-# else
-# error "please specify a target using the -bt compiler option"
-# endif
-#elif defined(__palmos__)
-# define LZO_OS_PALMOS 1
-# define LZO_INFO_OS "palmos"
-#elif defined(__TOS__) || defined(__atarist__)
-# define LZO_OS_TOS 1
-# define LZO_INFO_OS "tos"
-#elif defined(macintosh) && !defined(__arm__) && !defined(__i386__) && !defined(__ppc__) && !defined(__x64_64__)
-# define LZO_OS_MACCLASSIC 1
-# define LZO_INFO_OS "macclassic"
-#elif defined(__VMS)
-# define LZO_OS_VMS 1
-# define LZO_INFO_OS "vms"
-#elif (defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__)
-# define LZO_OS_CONSOLE 1
-# define LZO_OS_CONSOLE_PS2 1
-# define LZO_INFO_OS "console"
-# define LZO_INFO_OS_CONSOLE "ps2"
-#elif defined(__mips__) && defined(__psp__)
-# define LZO_OS_CONSOLE 1
-# define LZO_OS_CONSOLE_PSP 1
-# define LZO_INFO_OS "console"
-# define LZO_INFO_OS_CONSOLE "psp"
-#else
-# define LZO_OS_POSIX 1
-# define LZO_INFO_OS "posix"
-#endif
-#if (LZO_OS_POSIX)
-# if defined(_AIX) || defined(__AIX__) || defined(__aix__)
-# define LZO_OS_POSIX_AIX 1
-# define LZO_INFO_OS_POSIX "aix"
-# elif defined(__FreeBSD__)
-# define LZO_OS_POSIX_FREEBSD 1
-# define LZO_INFO_OS_POSIX "freebsd"
-# elif defined(__hpux__) || defined(__hpux)
-# define LZO_OS_POSIX_HPUX 1
-# define LZO_INFO_OS_POSIX "hpux"
-# elif defined(__INTERIX)
-# define LZO_OS_POSIX_INTERIX 1
-# define LZO_INFO_OS_POSIX "interix"
-# elif defined(__IRIX__) || defined(__irix__)
-# define LZO_OS_POSIX_IRIX 1
-# define LZO_INFO_OS_POSIX "irix"
-# elif defined(__linux__) || defined(__linux) || defined(__LINUX__)
-# define LZO_OS_POSIX_LINUX 1
-# define LZO_INFO_OS_POSIX "linux"
-# elif defined(__APPLE__) && defined(__MACH__)
-# if ((__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__-0) >= 20000)
-# define LZO_OS_POSIX_DARWIN 1040
-# define LZO_INFO_OS_POSIX "darwin_iphone"
-# elif ((__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__-0) >= 1040)
-# define LZO_OS_POSIX_DARWIN __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__
-# define LZO_INFO_OS_POSIX "darwin"
-# else
-# define LZO_OS_POSIX_DARWIN 1
-# define LZO_INFO_OS_POSIX "darwin"
-# endif
-# define LZO_OS_POSIX_MACOSX LZO_OS_POSIX_DARWIN
-# elif defined(__minix__) || defined(__minix)
-# define LZO_OS_POSIX_MINIX 1
-# define LZO_INFO_OS_POSIX "minix"
-# elif defined(__NetBSD__)
-# define LZO_OS_POSIX_NETBSD 1
-# define LZO_INFO_OS_POSIX "netbsd"
-# elif defined(__OpenBSD__)
-# define LZO_OS_POSIX_OPENBSD 1
-# define LZO_INFO_OS_POSIX "openbsd"
-# elif defined(__osf__)
-# define LZO_OS_POSIX_OSF 1
-# define LZO_INFO_OS_POSIX "osf"
-# elif defined(__solaris__) || defined(__sun)
-# if defined(__SVR4) || defined(__svr4__)
-# define LZO_OS_POSIX_SOLARIS 1
-# define LZO_INFO_OS_POSIX "solaris"
-# else
-# define LZO_OS_POSIX_SUNOS 1
-# define LZO_INFO_OS_POSIX "sunos"
-# endif
-# elif defined(__ultrix__) || defined(__ultrix)
-# define LZO_OS_POSIX_ULTRIX 1
-# define LZO_INFO_OS_POSIX "ultrix"
-# elif defined(_UNICOS)
-# define LZO_OS_POSIX_UNICOS 1
-# define LZO_INFO_OS_POSIX "unicos"
-# else
-# define LZO_OS_POSIX_UNKNOWN 1
-# define LZO_INFO_OS_POSIX "unknown"
-# endif
-#endif
-#endif
-#if (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-# if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_OS_DOS32 || LZO_OS_OS2 || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (UINT_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if defined(CIL) && defined(_GNUCC) && defined(__GNUC__)
-# define LZO_CC_CILLY 1
-# define LZO_INFO_CC "Cilly"
-# if defined(__CILLY__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__CILLY__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif 0 && defined(SDCC) && defined(__VERSION__) && !defined(__GNUC__)
-# define LZO_CC_SDCC 1
-# define LZO_INFO_CC "sdcc"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(SDCC)
-#elif defined(__PATHSCALE__) && defined(__PATHCC_PATCHLEVEL__)
-# define LZO_CC_PATHSCALE (__PATHCC__ * 0x10000L + (__PATHCC_MINOR__-0) * 0x100 + (__PATHCC_PATCHLEVEL__-0))
-# define LZO_INFO_CC "Pathscale C"
-# define LZO_INFO_CCVER __PATHSCALE__
-# if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_PATHSCALE_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__INTEL_COMPILER) && ((__INTEL_COMPILER-0) > 0)
-# define LZO_CC_INTELC __INTEL_COMPILER
-# define LZO_INFO_CC "Intel C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__INTEL_COMPILER)
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_INTELC_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_INTELC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__POCC__) && defined(_WIN32)
-# define LZO_CC_PELLESC 1
-# define LZO_INFO_CC "Pelles C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__POCC__)
-#elif defined(__ARMCC_VERSION) && defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# if defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_ARMCC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# else
-# define LZO_CC_ARMCC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# endif
-# define LZO_CC_ARMCC __ARMCC_VERSION
-# define LZO_INFO_CC "ARM C Compiler"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__clang__) && defined(__llvm__) && defined(__VERSION__)
-# if defined(__clang_major__) && defined(__clang_minor__) && defined(__clang_patchlevel__)
-# define LZO_CC_CLANG (__clang_major__ * 0x10000L + (__clang_minor__-0) * 0x100 + (__clang_patchlevel__-0))
-# else
-# define LZO_CC_CLANG 0x010000L
-# endif
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_CLANG_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_CLANG_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-# define LZO_INFO_CC "clang"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__llvm__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# if defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_LLVM_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# else
-# define LZO_CC_LLVM_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# endif
-# define LZO_CC_LLVM LZO_CC_LLVM_GNUC
-# define LZO_INFO_CC "llvm-gcc"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__ACK__) && defined(_ACK)
-# define LZO_CC_ACK 1
-# define LZO_INFO_CC "Amsterdam Compiler Kit C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__ARMCC_VERSION) && !defined(__GNUC__)
-# define LZO_CC_ARMCC __ARMCC_VERSION
-# define LZO_CC_ARMCC_ARMCC __ARMCC_VERSION
-# define LZO_INFO_CC "ARM C Compiler"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__ARMCC_VERSION)
-#elif defined(__AZTEC_C__)
-# define LZO_CC_AZTECC 1
-# define LZO_INFO_CC "Aztec C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__AZTEC_C__)
-#elif defined(__CODEGEARC__)
-# define LZO_CC_CODEGEARC 1
-# define LZO_INFO_CC "CodeGear C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__CODEGEARC__)
-#elif defined(__BORLANDC__)
-# define LZO_CC_BORLANDC 1
-# define LZO_INFO_CC "Borland C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__BORLANDC__)
-#elif defined(_CRAYC) && defined(_RELEASE)
-# define LZO_CC_CRAYC 1
-# define LZO_INFO_CC "Cray C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_RELEASE)
-#elif defined(__DMC__) && defined(__SC__)
-# define LZO_CC_DMC 1
-# define LZO_INFO_CC "Digital Mars C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__DMC__)
-#elif defined(__DECC)
-# define LZO_CC_DECC 1
-# define LZO_INFO_CC "DEC C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__DECC)
-#elif (defined(__ghs) || defined(__ghs__)) && defined(__GHS_VERSION_NUMBER) && ((__GHS_VERSION_NUMBER-0) > 0)
-# define LZO_CC_GHS 1
-# define LZO_INFO_CC "Green Hills C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__GHS_VERSION_NUMBER)
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_GHS_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_GHS_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__HIGHC__)
-# define LZO_CC_HIGHC 1
-# define LZO_INFO_CC "MetaWare High C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__HP_aCC) && ((__HP_aCC-0) > 0)
-# define LZO_CC_HPACC __HP_aCC
-# define LZO_INFO_CC "HP aCC"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__HP_aCC)
-#elif defined(__IAR_SYSTEMS_ICC__)
-# define LZO_CC_IARC 1
-# define LZO_INFO_CC "IAR C"
-# if defined(__VER__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__VER__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__IBMC__) && ((__IBMC__-0) > 0)
-# define LZO_CC_IBMC __IBMC__
-# define LZO_INFO_CC "IBM C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__IBMC__)
-#elif defined(__IBMCPP__) && ((__IBMCPP__-0) > 0)
-# define LZO_CC_IBMC __IBMCPP__
-# define LZO_INFO_CC "IBM C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__IBMCPP__)
-#elif defined(__KEIL__) && defined(__C166__)
-# define LZO_CC_KEILC 1
-# define LZO_INFO_CC "Keil C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__C166__)
-#elif defined(__LCC__) && defined(_WIN32) && defined(__LCCOPTIMLEVEL)
-# define LZO_CC_LCCWIN32 1
-# define LZO_INFO_CC "lcc-win32"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__LCC__)
-# define LZO_CC_LCC 1
-# define LZO_INFO_CC "lcc"
-# if defined(__LCC_VERSION__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__LCC_VERSION__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__MWERKS__) && ((__MWERKS__-0) > 0)
-# define LZO_CC_MWERKS __MWERKS__
-# define LZO_INFO_CC "Metrowerks C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__MWERKS__)
-#elif (defined(__NDPC__) || defined(__NDPX__)) && defined(__i386)
-# define LZO_CC_NDPC 1
-# define LZO_INFO_CC "Microway NDP C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__PACIFIC__)
-# define LZO_CC_PACIFICC 1
-# define LZO_INFO_CC "Pacific C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PACIFIC__)
-#elif defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__)
-# if defined(__PGIC_PATCHLEVEL__)
-# define LZO_CC_PGI (__PGIC__ * 0x10000L + (__PGIC_MINOR__-0) * 0x100 + (__PGIC_PATCHLEVEL__-0))
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PGIC__) "." LZO_PP_MACRO_EXPAND(__PGIC_MINOR__) "." LZO_PP_MACRO_EXPAND(__PGIC_PATCHLEVEL__)
-# else
-# define LZO_CC_PGI (__PGIC__ * 0x10000L + (__PGIC_MINOR__-0) * 0x100)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PGIC__) "." LZO_PP_MACRO_EXPAND(__PGIC_MINOR__) ".0"
-# endif
-# define LZO_INFO_CC "Portland Group PGI C"
-#elif defined(__PGI) && (defined(__linux__) || defined(__WIN32__))
-# define LZO_CC_PGI 1
-# define LZO_INFO_CC "Portland Group PGI C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__PUREC__) && defined(__TOS__)
-# define LZO_CC_PUREC 1
-# define LZO_INFO_CC "Pure C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PUREC__)
-#elif defined(__SC__) && defined(__ZTC__)
-# define LZO_CC_SYMANTECC 1
-# define LZO_INFO_CC "Symantec C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SC__)
-#elif defined(__SUNPRO_C)
-# define LZO_INFO_CC "SunPro C"
-# if ((__SUNPRO_C-0) > 0)
-# define LZO_CC_SUNPROC __SUNPRO_C
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SUNPRO_C)
-# else
-# define LZO_CC_SUNPROC 1
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__SUNPRO_CC)
-# define LZO_INFO_CC "SunPro C"
-# if ((__SUNPRO_CC-0) > 0)
-# define LZO_CC_SUNPROC __SUNPRO_CC
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SUNPRO_CC)
-# else
-# define LZO_CC_SUNPROC 1
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__TINYC__)
-# define LZO_CC_TINYC 1
-# define LZO_INFO_CC "Tiny C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TINYC__)
-#elif defined(__TSC__)
-# define LZO_CC_TOPSPEEDC 1
-# define LZO_INFO_CC "TopSpeed C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TSC__)
-#elif defined(__WATCOMC__)
-# define LZO_CC_WATCOMC 1
-# define LZO_INFO_CC "Watcom C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__WATCOMC__)
-#elif defined(__TURBOC__)
-# define LZO_CC_TURBOC 1
-# define LZO_INFO_CC "Turbo C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TURBOC__)
-#elif defined(__ZTC__)
-# define LZO_CC_ZORTECHC 1
-# define LZO_INFO_CC "Zortech C"
-# if ((__ZTC__-0) == 0x310)
-# define LZO_INFO_CCVER "0x310"
-# else
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__ZTC__)
-# endif
-#elif defined(__GNUC__) && defined(__VERSION__)
-# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# elif defined(__GNUC_MINOR__)
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# else
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L)
-# endif
-# define LZO_INFO_CC "gcc"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_MSC _MSC_VER
-# define LZO_INFO_CC "Microsoft C"
-# if defined(_MSC_FULL_VER)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_MSC_VER) "." LZO_PP_MACRO_EXPAND(_MSC_FULL_VER)
-# else
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_MSC_VER)
-# endif
-#else
-# define LZO_CC_UNKNOWN 1
-# define LZO_INFO_CC "unknown"
-# define LZO_INFO_CCVER "unknown"
-#endif
-#if (LZO_CC_GNUC) && defined(__OPEN64__)
-# if defined(__OPENCC__) && defined(__OPENCC_MINOR__) && defined(__OPENCC_PATCHLEVEL__)
-# define LZO_CC_OPEN64 (__OPENCC__ * 0x10000L + (__OPENCC_MINOR__-0) * 0x100 + (__OPENCC_PATCHLEVEL__-0))
-# define LZO_CC_OPEN64_GNUC LZO_CC_GNUC
-# endif
-#endif
-#if (LZO_CC_GNUC) && defined(__PCC__)
-# if defined(__PCC__) && defined(__PCC_MINOR__) && defined(__PCC_MINORMINOR__)
-# define LZO_CC_PCC (__PCC__ * 0x10000L + (__PCC_MINOR__-0) * 0x100 + (__PCC_MINORMINOR__-0))
-# define LZO_CC_PCC_GNUC LZO_CC_GNUC
-# endif
-#endif
-#if 0 && (LZO_CC_MSC && (_MSC_VER >= 1200)) && !defined(_MSC_FULL_VER)
-# error "LZO_CC_MSC: _MSC_FULL_VER is not defined"
-#endif
-#if !defined(__LZO_ARCH_OVERRIDE) && !(LZO_ARCH_GENERIC) && defined(_CRAY)
-# if (UINT_MAX > LZO_0xffffffffL) && defined(_CRAY)
-# if defined(_CRAYMPP) || defined(_CRAYT3D) || defined(_CRAYT3E)
-# define LZO_ARCH_CRAY_MPP 1
-# elif defined(_CRAY1)
-# define LZO_ARCH_CRAY_PVP 1
-# endif
-# endif
-#endif
-#if !defined(__LZO_ARCH_OVERRIDE)
-#if (LZO_ARCH_GENERIC)
-# define LZO_INFO_ARCH "generic"
-#elif (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-# define LZO_ARCH_I086 1
-# define LZO_INFO_ARCH "i086"
-#elif defined(__aarch64__)
-# define LZO_ARCH_ARM64 1
-# define LZO_INFO_ARCH "arm64"
-#elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA)
-# define LZO_ARCH_ALPHA 1
-# define LZO_INFO_ARCH "alpha"
-#elif (LZO_ARCH_CRAY_MPP) && (defined(_CRAYT3D) || defined(_CRAYT3E))
-# define LZO_ARCH_ALPHA 1
-# define LZO_INFO_ARCH "alpha"
-#elif defined(__amd64__) || defined(__x86_64__) || defined(_M_AMD64)
-# define LZO_ARCH_AMD64 1
-# define LZO_INFO_ARCH "amd64"
-#elif defined(__arm__) || defined(_M_ARM)
-# define LZO_ARCH_ARM 1
-# define LZO_INFO_ARCH "arm"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCARM__)
-# define LZO_ARCH_ARM 1
-# define LZO_INFO_ARCH "arm"
-#elif (UINT_MAX <= LZO_0xffffL) && defined(__AVR__)
-# define LZO_ARCH_AVR 1
-# define LZO_INFO_ARCH "avr"
-#elif defined(__avr32__) || defined(__AVR32__)
-# define LZO_ARCH_AVR32 1
-# define LZO_INFO_ARCH "avr32"
-#elif defined(__bfin__)
-# define LZO_ARCH_BLACKFIN 1
-# define LZO_INFO_ARCH "blackfin"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C166__)
-# define LZO_ARCH_C166 1
-# define LZO_INFO_ARCH "c166"
-#elif defined(__cris__)
-# define LZO_ARCH_CRIS 1
-# define LZO_INFO_ARCH "cris"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCEZ80__)
-# define LZO_ARCH_EZ80 1
-# define LZO_INFO_ARCH "ez80"
-#elif defined(__H8300__) || defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_ARCH_H8300 1
-# define LZO_INFO_ARCH "h8300"
-#elif defined(__hppa__) || defined(__hppa)
-# define LZO_ARCH_HPPA 1
-# define LZO_INFO_ARCH "hppa"
-#elif defined(__386__) || defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_M_I386)
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif (LZO_CC_ZORTECHC && defined(__I86__))
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif (LZO_OS_DOS32 && LZO_CC_HIGHC) && defined(_I386)
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
-# define LZO_ARCH_IA64 1
-# define LZO_INFO_ARCH "ia64"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__m32c__)
-# define LZO_ARCH_M16C 1
-# define LZO_INFO_ARCH "m16c"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCM16C__)
-# define LZO_ARCH_M16C 1
-# define LZO_INFO_ARCH "m16c"
-#elif defined(__m32r__)
-# define LZO_ARCH_M32R 1
-# define LZO_INFO_ARCH "m32r"
-#elif (LZO_OS_TOS) || defined(__m68k__) || defined(__m68000__) || defined(__mc68000__) || defined(__mc68020__) || defined(_M_M68K)
-# define LZO_ARCH_M68K 1
-# define LZO_INFO_ARCH "m68k"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C251__)
-# define LZO_ARCH_MCS251 1
-# define LZO_INFO_ARCH "mcs251"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C51__)
-# define LZO_ARCH_MCS51 1
-# define LZO_INFO_ARCH "mcs51"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICC8051__)
-# define LZO_ARCH_MCS51 1
-# define LZO_INFO_ARCH "mcs51"
-#elif defined(__mips__) || defined(__mips) || defined(_MIPS_ARCH) || defined(_M_MRX000)
-# define LZO_ARCH_MIPS 1
-# define LZO_INFO_ARCH "mips"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__MSP430__)
-# define LZO_ARCH_MSP430 1
-# define LZO_INFO_ARCH "msp430"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICC430__)
-# define LZO_ARCH_MSP430 1
-# define LZO_INFO_ARCH "msp430"
-#elif defined(__powerpc__) || defined(__powerpc) || defined(__ppc__) || defined(__PPC__) || defined(_M_PPC) || defined(_ARCH_PPC) || defined(_ARCH_PWR)
-# define LZO_ARCH_POWERPC 1
-# define LZO_INFO_ARCH "powerpc"
-#elif defined(__s390__) || defined(__s390) || defined(__s390x__) || defined(__s390x)
-# define LZO_ARCH_S390 1
-# define LZO_INFO_ARCH "s390"
-#elif defined(__sh__) || defined(_M_SH)
-# define LZO_ARCH_SH 1
-# define LZO_INFO_ARCH "sh"
-#elif defined(__sparc__) || defined(__sparc) || defined(__sparcv8)
-# define LZO_ARCH_SPARC 1
-# define LZO_INFO_ARCH "sparc"
-#elif defined(__SPU__)
-# define LZO_ARCH_SPU 1
-# define LZO_INFO_ARCH "spu"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__z80)
-# define LZO_ARCH_Z80 1
-# define LZO_INFO_ARCH "z80"
-#elif (LZO_ARCH_CRAY_PVP)
-# if defined(_CRAYSV1)
-# define LZO_ARCH_CRAY_SV1 1
-# define LZO_INFO_ARCH "cray_sv1"
-# elif (_ADDR64)
-# define LZO_ARCH_CRAY_T90 1
-# define LZO_INFO_ARCH "cray_t90"
-# elif (_ADDR32)
-# define LZO_ARCH_CRAY_YMP 1
-# define LZO_INFO_ARCH "cray_ymp"
-# else
-# define LZO_ARCH_CRAY_XMP 1
-# define LZO_INFO_ARCH "cray_xmp"
-# endif
-#else
-# define LZO_ARCH_UNKNOWN 1
-# define LZO_INFO_ARCH "unknown"
-#endif
-#endif
-#if !defined(LZO_ARCH_ARM_THUMB2)
-#if (LZO_ARCH_ARM)
-# if defined(__ARM_ARCH_ISA_THUMB)
-# if ((__ARM_ARCH_ISA_THUMB)+0 >= 2)
-# define LZO_ARCH_ARM_THUMB2 1
-# endif
-# elif 1 && defined(__thumb2__)
-# define LZO_ARCH_ARM_THUMB2 1
-# elif 1 && defined(__TARGET_ARCH_THUMB) && ((__TARGET_ARCH_THUMB)+0 >= 4)
-# define LZO_ARCH_ARM_THUMB2 1
-# endif
-#endif
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_DOS32 || LZO_OS_OS2)
-# error "FIXME - missing define for CPU architecture"
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_WIN32)
-# error "FIXME - missing LZO_OS_WIN32 define for CPU architecture"
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_WIN64)
-# error "FIXME - missing LZO_OS_WIN64 define for CPU architecture"
-#endif
-#if (LZO_OS_OS216 || LZO_OS_WIN16)
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && defined(BLX286))
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && defined(DOSX286))
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && LZO_CC_BORLANDC && defined(__DPMI16__))
-# define LZO_ARCH_I086PM 1
-#endif
-#if (LZO_ARCH_AMD64 && !LZO_ARCH_X64)
-# define LZO_ARCH_X64 1
-#elif (!LZO_ARCH_AMD64 && LZO_ARCH_X64) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_AMD64 1
-#endif
-#if (LZO_ARCH_ARM64 && !LZO_ARCH_AARCH64)
-# define LZO_ARCH_AARCH64 1
-#elif (!LZO_ARCH_ARM64 && LZO_ARCH_AARCH64) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_ARM64 1
-#endif
-#if (LZO_ARCH_I386 && !LZO_ARCH_X86)
-# define LZO_ARCH_X86 1
-#elif (!LZO_ARCH_I386 && LZO_ARCH_X86) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_I386 1
-#endif
-#if (LZO_ARCH_AMD64 && !LZO_ARCH_X64) || (!LZO_ARCH_AMD64 && LZO_ARCH_X64)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM64 && !LZO_ARCH_AARCH64) || (!LZO_ARCH_ARM64 && LZO_ARCH_AARCH64)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I386 && !LZO_ARCH_X86) || (!LZO_ARCH_I386 && LZO_ARCH_X86)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB1 && !LZO_ARCH_ARM)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB2 && !LZO_ARCH_ARM)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB1 && LZO_ARCH_ARM_THUMB2)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I086PM && !LZO_ARCH_I086)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I086)
-# if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_ARCH_I386)
-# if (UINT_MAX != LZO_0xffffL) && defined(__i386_int16__)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (UINT_MAX != LZO_0xffffffffL) && !defined(__i386_int16__)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-# if !defined(LZO_TARGET_FEATURE_SSE2)
-# if defined(__SSE2__)
-# define LZO_TARGET_FEATURE_SSE2 1
-# elif defined(_MSC_VER) && (defined(_M_IX86_FP) && ((_M_IX86_FP)+0 >= 2))
-# define LZO_TARGET_FEATURE_SSE2 1
-# elif (LZO_CC_INTELC_MSC || LZO_CC_MSC) && defined(_M_AMD64)
-# define LZO_TARGET_FEATURE_SSE2 1
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_SSSE3)
-# if (LZO_TARGET_FEATURE_SSE2)
-# if defined(__SSSE3__)
-# define LZO_TARGET_FEATURE_SSSE3 1
-# elif defined(_MSC_VER) && defined(__AVX__)
-# define LZO_TARGET_FEATURE_SSSE3 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_SSE4_2)
-# if (LZO_TARGET_FEATURE_SSSE3)
-# if defined(__SSE4_2__)
-# define LZO_TARGET_FEATURE_SSE4_2 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_AVX)
-# if (LZO_TARGET_FEATURE_SSSE3)
-# if defined(__AVX__)
-# define LZO_TARGET_FEATURE_AVX 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_AVX2)
-# if (LZO_TARGET_FEATURE_AVX)
-# if defined(__AVX2__)
-# define LZO_TARGET_FEATURE_AVX2 1
-# endif
-# endif
-# endif
-#endif
-#if (LZO_TARGET_FEATURE_SSSE3 && !(LZO_TARGET_FEATURE_SSE2))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_SSE4_2 && !(LZO_TARGET_FEATURE_SSSE3))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_AVX && !(LZO_TARGET_FEATURE_SSSE3))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_AVX2 && !(LZO_TARGET_FEATURE_AVX))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM)
-# if !defined(LZO_TARGET_FEATURE_NEON)
-# if defined(__ARM_NEON) && ((__ARM_NEON)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# elif 1 && defined(__ARM_NEON__) && ((__ARM_NEON__)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# elif 1 && defined(__TARGET_FEATURE_NEON) && ((__TARGET_FEATURE_NEON)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# endif
-# endif
-#elif (LZO_ARCH_ARM64)
-# if !defined(LZO_TARGET_FEATURE_NEON)
-# if 1
-# define LZO_TARGET_FEATURE_NEON 1
-# endif
-# endif
-#endif
-#if 0
-#elif !defined(__LZO_MM_OVERRIDE)
-#if (LZO_ARCH_I086)
-#if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if defined(__TINY__) || defined(M_I86TM) || defined(_M_I86TM)
-# define LZO_MM_TINY 1
-#elif defined(__HUGE__) || defined(_HUGE_) || defined(M_I86HM) || defined(_M_I86HM)
-# define LZO_MM_HUGE 1
-#elif defined(__SMALL__) || defined(M_I86SM) || defined(_M_I86SM) || defined(SMALL_MODEL)
-# define LZO_MM_SMALL 1
-#elif defined(__MEDIUM__) || defined(M_I86MM) || defined(_M_I86MM)
-# define LZO_MM_MEDIUM 1
-#elif defined(__COMPACT__) || defined(M_I86CM) || defined(_M_I86CM)
-# define LZO_MM_COMPACT 1
-#elif defined(__LARGE__) || defined(M_I86LM) || defined(_M_I86LM) || defined(LARGE_MODEL)
-# define LZO_MM_LARGE 1
-#elif (LZO_CC_AZTECC)
-# if defined(_LARGE_CODE) && defined(_LARGE_DATA)
-# define LZO_MM_LARGE 1
-# elif defined(_LARGE_CODE)
-# define LZO_MM_MEDIUM 1
-# elif defined(_LARGE_DATA)
-# define LZO_MM_COMPACT 1
-# else
-# define LZO_MM_SMALL 1
-# endif
-#elif (LZO_CC_ZORTECHC && defined(__VCM__))
-# define LZO_MM_LARGE 1
-#else
-# error "unknown LZO_ARCH_I086 memory model"
-#endif
-#if (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-#define LZO_HAVE_MM_HUGE_PTR 1
-#define LZO_HAVE_MM_HUGE_ARRAY 1
-#if (LZO_MM_TINY)
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#endif
-#if (LZO_CC_AZTECC || LZO_CC_PACIFICC || LZO_CC_ZORTECHC)
-# undef LZO_HAVE_MM_HUGE_PTR
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#elif (LZO_CC_DMC || LZO_CC_SYMANTECC)
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#elif (LZO_CC_MSC && defined(_QC))
-# undef LZO_HAVE_MM_HUGE_ARRAY
-# if (_MSC_VER < 600)
-# undef LZO_HAVE_MM_HUGE_PTR
-# endif
-#elif (LZO_CC_TURBOC && (__TURBOC__ < 0x0295))
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#endif
-#if (LZO_ARCH_I086PM) && !(LZO_HAVE_MM_HUGE_PTR)
-# if (LZO_OS_DOS16)
-# error "unexpected configuration - check your compiler defines"
-# elif (LZO_CC_ZORTECHC)
-# else
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0200))
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_DMC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_MSC || LZO_CC_TOPSPEEDC)
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_TURBOC && (__TURBOC__ >= 0x0295))
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif ((LZO_CC_AZTECC || LZO_CC_PACIFICC || LZO_CC_TURBOC) && LZO_OS_DOS16)
-# define LZO_MM_AHSHIFT 12
-#elif (LZO_CC_WATCOMC)
- extern unsigned char _HShift;
-# define LZO_MM_AHSHIFT ((unsigned) _HShift)
-#else
-# error "FIXME - implement LZO_MM_AHSHIFT"
-#endif
-#ifdef __cplusplus
-}
-#endif
-#endif
-#elif (LZO_ARCH_C166)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_C166 __MODEL__"
-#elif ((__MODEL__) == 0)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 1)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_C166 __MODEL__"
-#endif
-#elif (LZO_ARCH_MCS251)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_MCS251 __MODEL__"
-#elif ((__MODEL__) == 0)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_MCS251 __MODEL__"
-#endif
-#elif (LZO_ARCH_MCS51)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_MCS51 __MODEL__"
-#elif ((__MODEL__) == 1)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_MCS51 __MODEL__"
-#endif
-#elif (LZO_ARCH_CRAY_PVP)
-# define LZO_MM_PVP 1
-#else
-# define LZO_MM_FLAT 1
-#endif
-#if (LZO_MM_COMPACT)
-# define LZO_INFO_MM "compact"
-#elif (LZO_MM_FLAT)
-# define LZO_INFO_MM "flat"
-#elif (LZO_MM_HUGE)
-# define LZO_INFO_MM "huge"
-#elif (LZO_MM_LARGE)
-# define LZO_INFO_MM "large"
-#elif (LZO_MM_MEDIUM)
-# define LZO_INFO_MM "medium"
-#elif (LZO_MM_PVP)
-# define LZO_INFO_MM "pvp"
-#elif (LZO_MM_SMALL)
-# define LZO_INFO_MM "small"
-#elif (LZO_MM_TINY)
-# define LZO_INFO_MM "tiny"
-#else
-# error "unknown memory model"
-#endif
-#endif
-#if !defined(__lzo_gnuc_extension__)
-#if (LZO_CC_GNUC >= 0x020800ul)
-# define __lzo_gnuc_extension__ __extension__
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_gnuc_extension__ __extension__
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_gnuc_extension__ __extension__
-#else
-#endif
-#endif
-#if !defined(__lzo_gnuc_extension__)
-# define __lzo_gnuc_extension__ /*empty*/
-#endif
-#if !defined(lzo_has_builtin)
-#if (LZO_CC_CLANG) && defined(__has_builtin)
-# define lzo_has_builtin __has_builtin
-#endif
-#endif
-#if !defined(lzo_has_builtin)
-# define lzo_has_builtin(x) 0
-#endif
-#if !defined(lzo_has_attribute)
-#if (LZO_CC_CLANG) && defined(__has_attribute)
-# define lzo_has_attribute __has_attribute
-#endif
-#endif
-#if !defined(lzo_has_attribute)
-# define lzo_has_attribute(x) 0
-#endif
-#if !defined(lzo_has_declspec_attribute)
-#if (LZO_CC_CLANG) && defined(__has_declspec_attribute)
-# define lzo_has_declspec_attribute __has_declspec_attribute
-#endif
-#endif
-#if !defined(lzo_has_declspec_attribute)
-# define lzo_has_declspec_attribute(x) 0
-#endif
-#if !defined(lzo_has_feature)
-#if (LZO_CC_CLANG) && defined(__has_feature)
-# define lzo_has_feature __has_feature
-#endif
-#endif
-#if !defined(lzo_has_feature)
-# define lzo_has_feature(x) 0
-#endif
-#if !defined(lzo_has_extension)
-#if (LZO_CC_CLANG) && defined(__has_extension)
-# define lzo_has_extension __has_extension
-#elif (LZO_CC_CLANG) && defined(__has_feature)
-# define lzo_has_extension __has_feature
-#endif
-#endif
-#if !defined(lzo_has_extension)
-# define lzo_has_extension 0
-#endif
-#if !defined(LZO_CFG_USE_NEW_STYLE_CASTS) && defined(__cplusplus) && 0
-# if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul))
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-# elif (LZO_CC_INTELC && (__INTEL_COMPILER < 1200))
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-# else
-# define LZO_CFG_USE_NEW_STYLE_CASTS 1
-# endif
-#endif
-#if !defined(LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-#endif
-#if !defined(__cplusplus)
-# if defined(LZO_CFG_USE_NEW_STYLE_CASTS)
-# undef LZO_CFG_USE_NEW_STYLE_CASTS
-# endif
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-#endif
-#if !defined(LZO_REINTERPRET_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_REINTERPRET_CAST(t,e) (reinterpret_cast<t> (e))
-# endif
-#endif
-#if !defined(LZO_REINTERPRET_CAST)
-# define LZO_REINTERPRET_CAST(t,e) ((t) (e))
-#endif
-#if !defined(LZO_STATIC_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_STATIC_CAST(t,e) (static_cast<t> (e))
-# endif
-#endif
-#if !defined(LZO_STATIC_CAST)
-# define LZO_STATIC_CAST(t,e) ((t) (e))
-#endif
-#if !defined(LZO_STATIC_CAST2)
-# define LZO_STATIC_CAST2(t1,t2,e) LZO_STATIC_CAST(t1, LZO_STATIC_CAST(t2, e))
-#endif
-#if !defined(LZO_UNCONST_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNCONST_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNCONST_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNCONST_CAST(t,e) ((t) ((void *) ((lzo_uintptr_t) ((const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNCONST_CAST)
-# define LZO_UNCONST_CAST(t,e) ((t) ((void *) ((const void *) (e))))
-#endif
-#if !defined(LZO_UNCONST_VOLATILE_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) ((volatile void *) ((lzo_uintptr_t) ((volatile const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNCONST_VOLATILE_CAST)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) ((volatile void *) ((volatile const void *) (e))))
-#endif
-#if !defined(LZO_UNVOLATILE_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNVOLATILE_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) ((void *) ((lzo_uintptr_t) ((volatile void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNVOLATILE_CAST)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) ((void *) ((volatile void *) (e))))
-#endif
-#if !defined(LZO_UNVOLATILE_CONST_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) ((const void *) ((lzo_uintptr_t) ((volatile const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNVOLATILE_CONST_CAST)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) ((const void *) ((volatile const void *) (e))))
-#endif
-#if !defined(LZO_PCAST)
-# if (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_PCAST(t,e) ((t) (e))
-# endif
-#endif
-#if !defined(LZO_PCAST)
-# define LZO_PCAST(t,e) LZO_STATIC_CAST(t, LZO_STATIC_CAST(void *, e))
-#endif
-#if !defined(LZO_CCAST)
-# if (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_CCAST(t,e) ((t) (e))
-# endif
-#endif
-#if !defined(LZO_CCAST)
-# define LZO_CCAST(t,e) LZO_STATIC_CAST(t, LZO_STATIC_CAST(const void *, e))
-#endif
-#if !defined(LZO_ICONV)
-# define LZO_ICONV(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(LZO_ICAST)
-# define LZO_ICAST(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(LZO_ITRUNC)
-# define LZO_ITRUNC(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(__lzo_cte)
-# if (LZO_CC_MSC || LZO_CC_WATCOMC)
-# define __lzo_cte(e) ((void)0,(e))
-# elif 1
-# define __lzo_cte(e) ((void)0,(e))
-# endif
-#endif
-#if !defined(__lzo_cte)
-# define __lzo_cte(e) (e)
-#endif
-#if !defined(LZO_BLOCK_BEGIN)
-# define LZO_BLOCK_BEGIN do {
-# define LZO_BLOCK_END } while __lzo_cte(0)
-#endif
-#if !defined(LZO_UNUSED)
-# if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0600))
-# define LZO_UNUSED(var) ((void) &var)
-# elif (LZO_CC_BORLANDC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PELLESC || LZO_CC_TURBOC)
-# define LZO_UNUSED(var) if (&var) ; else
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x030200ul))
-# define LZO_UNUSED(var) ((void) &var)
-# elif (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNUSED(var) ((void) var)
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_UNUSED(var) if (&var) ; else
-# elif (LZO_CC_KEILC)
-# define LZO_UNUSED(var) {extern int lzo_unused__[1-2*!(sizeof(var)>0)]; (void)lzo_unused__;}
-# elif (LZO_CC_PACIFICC)
-# define LZO_UNUSED(var) ((void) sizeof(var))
-# elif (LZO_CC_WATCOMC) && defined(__cplusplus)
-# define LZO_UNUSED(var) ((void) var)
-# else
-# define LZO_UNUSED(var) ((void) &var)
-# endif
-#endif
-#if !defined(LZO_UNUSED_FUNC)
-# if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0600))
-# define LZO_UNUSED_FUNC(func) ((void) func)
-# elif (LZO_CC_BORLANDC || LZO_CC_NDPC || LZO_CC_TURBOC)
-# define LZO_UNUSED_FUNC(func) if (func) ; else
-# elif (LZO_CC_CLANG || LZO_CC_LLVM)
-# define LZO_UNUSED_FUNC(func) ((void) &func)
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_UNUSED_FUNC(func) if (func) ; else
-# elif (LZO_CC_MSC)
-# define LZO_UNUSED_FUNC(func) ((void) &func)
-# elif (LZO_CC_KEILC || LZO_CC_PELLESC)
-# define LZO_UNUSED_FUNC(func) {extern int lzo_unused__[1-2*!(sizeof((int)func)>0)]; (void)lzo_unused__;}
-# else
-# define LZO_UNUSED_FUNC(func) ((void) func)
-# endif
-#endif
-#if !defined(LZO_UNUSED_LABEL)
-# if (LZO_CC_CLANG >= 0x020800ul)
-# define LZO_UNUSED_LABEL(l) (__lzo_gnuc_extension__ ((void) ((const void *) &&l)))
-# elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_INTELC || LZO_CC_WATCOMC)
-# define LZO_UNUSED_LABEL(l) if __lzo_cte(0) goto l
-# else
-# define LZO_UNUSED_LABEL(l) switch (0) case 1:goto l
-# endif
-#endif
-#if !defined(LZO_DEFINE_UNINITIALIZED_VAR)
-# if 0
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var
-# elif 0 && (LZO_CC_GNUC)
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var = var
-# else
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var = init
-# endif
-#endif
-#if !defined(__lzo_inline)
-#if (LZO_CC_TURBOC && (__TURBOC__ <= 0x0295))
-#elif defined(__cplusplus)
-# define __lzo_inline inline
-#elif defined(__STDC_VERSION__) && (__STDC_VERSION__-0 >= 199901L)
-# define __lzo_inline inline
-#elif (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0550))
-# define __lzo_inline __inline
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define __lzo_inline __inline__
-#elif (LZO_CC_DMC)
-# define __lzo_inline __inline
-#elif (LZO_CC_GHS)
-# define __lzo_inline __inline__
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_inline __inline__
-#elif (LZO_CC_INTELC)
-# define __lzo_inline __inline
-#elif (LZO_CC_MWERKS && (__MWERKS__ >= 0x2405))
-# define __lzo_inline __inline
-#elif (LZO_CC_MSC && (_MSC_VER >= 900))
-# define __lzo_inline __inline
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_inline __inline__
-#endif
-#endif
-#if defined(__lzo_inline)
-# ifndef __lzo_HAVE_inline
-# define __lzo_HAVE_inline 1
-# endif
-#else
-# define __lzo_inline /*empty*/
-#endif
-#if !defined(__lzo_forceinline)
-#if (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450))
-# define __lzo_forceinline __forceinline
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200))
-# define __lzo_forceinline __forceinline
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#endif
-#endif
-#if defined(__lzo_forceinline)
-# ifndef __lzo_HAVE_forceinline
-# define __lzo_HAVE_forceinline 1
-# endif
-#else
-# define __lzo_forceinline __lzo_inline
-#endif
-#if !defined(__lzo_noinline)
-#if 1 && (LZO_ARCH_I386) && (LZO_CC_GNUC >= 0x040000ul) && (LZO_CC_GNUC < 0x040003ul)
-# define __lzo_noinline __attribute__((__noinline__,__used__))
-#elif (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 600))
-# define __lzo_noinline __declspec(noinline)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_noinline __declspec(noinline)
-#elif (LZO_CC_MWERKS && (__MWERKS__ >= 0x3200) && (LZO_OS_WIN32 || LZO_OS_WIN64))
-# if defined(__cplusplus)
-# else
-# define __lzo_noinline __declspec(noinline)
-# endif
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_noinline __attribute__((__noinline__))
-#endif
-#endif
-#if defined(__lzo_noinline)
-# ifndef __lzo_HAVE_noinline
-# define __lzo_HAVE_noinline 1
-# endif
-#else
-# define __lzo_noinline /*empty*/
-#endif
-#if (__lzo_HAVE_forceinline || __lzo_HAVE_noinline) && !(__lzo_HAVE_inline)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if !defined(__lzo_static_inline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_inline __lzo_gnuc_extension__ static __lzo_inline
-#endif
-#endif
-#if !defined(__lzo_static_inline)
-# define __lzo_static_inline static __lzo_inline
-#endif
-#if !defined(__lzo_static_forceinline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_forceinline __lzo_gnuc_extension__ static __lzo_forceinline
-#endif
-#endif
-#if !defined(__lzo_static_forceinline)
-# define __lzo_static_forceinline static __lzo_forceinline
-#endif
-#if !defined(__lzo_static_noinline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_noinline __lzo_gnuc_extension__ static __lzo_noinline
-#endif
-#endif
-#if !defined(__lzo_static_noinline)
-# define __lzo_static_noinline static __lzo_noinline
-#endif
-#if !defined(__lzo_c99_extern_inline)
-#if defined(__GNUC_GNU_INLINE__)
-# define __lzo_c99_extern_inline __lzo_inline
-#elif defined(__GNUC_STDC_INLINE__)
-# define __lzo_c99_extern_inline extern __lzo_inline
-#elif defined(__STDC_VERSION__) && (__STDC_VERSION__-0 >= 199901L)
-# define __lzo_c99_extern_inline extern __lzo_inline
-#endif
-#if !defined(__lzo_c99_extern_inline) && (__lzo_HAVE_inline)
-# define __lzo_c99_extern_inline __lzo_inline
-#endif
-#endif
-#if defined(__lzo_c99_extern_inline)
-# ifndef __lzo_HAVE_c99_extern_inline
-# define __lzo_HAVE_c99_extern_inline 1
-# endif
-#else
-# define __lzo_c99_extern_inline /*empty*/
-#endif
-#if !defined(__lzo_may_alias)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_CLANG >= 0x020900ul)
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 1210)) && 0
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_PGI >= 0x0d0a00ul) && 0
-# define __lzo_may_alias __attribute__((__may_alias__))
-#endif
-#endif
-#if defined(__lzo_may_alias)
-# ifndef __lzo_HAVE_may_alias
-# define __lzo_HAVE_may_alias 1
-# endif
-#else
-# define __lzo_may_alias /*empty*/
-#endif
-#if !defined(__lzo_noreturn)
-#if (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450))
-# define __lzo_noreturn __declspec(noreturn)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 600))
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200))
-# define __lzo_noreturn __declspec(noreturn)
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#endif
-#endif
-#if defined(__lzo_noreturn)
-# ifndef __lzo_HAVE_noreturn
-# define __lzo_HAVE_noreturn 1
-# endif
-#else
-# define __lzo_noreturn /*empty*/
-#endif
-#if !defined(__lzo_nothrow)
-#if (LZO_CC_GNUC >= 0x030300ul)
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450)) && defined(__cplusplus)
-# define __lzo_nothrow __declspec(nothrow)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 900))
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200)) && defined(__cplusplus)
-# define __lzo_nothrow __declspec(nothrow)
-#endif
-#endif
-#if defined(__lzo_nothrow)
-# ifndef __lzo_HAVE_nothrow
-# define __lzo_HAVE_nothrow 1
-# endif
-#else
-# define __lzo_nothrow /*empty*/
-#endif
-#if !defined(__lzo_restrict)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_IBMC >= 800) && !defined(__cplusplus)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_IBMC >= 1210)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 600))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 600))
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_MSC && (_MSC_VER >= 1400))
-# define __lzo_restrict __restrict
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_restrict __restrict__
-#endif
-#endif
-#if defined(__lzo_restrict)
-# ifndef __lzo_HAVE_restrict
-# define __lzo_HAVE_restrict 1
-# endif
-#else
-# define __lzo_restrict /*empty*/
-#endif
-#if !defined(__lzo_alignof)
-#if (LZO_CC_ARMCC || LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_GHS) && !defined(__cplusplus)
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_alignof(e) (__lzo_gnuc_extension__ __alignof__(e))
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER >= 700))
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_alignof(e) __alignof(e)
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_alignof(e) __alignof__(e)
-#endif
-#endif
-#if defined(__lzo_alignof)
-# ifndef __lzo_HAVE_alignof
-# define __lzo_HAVE_alignof 1
-# endif
-#endif
-#if !defined(__lzo_struct_packed)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020700ul))
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_PCC && (LZO_CC_PCC < 0x010100ul))
-#elif (LZO_CC_SUNPROC && (LZO_CC_SUNPROC < 0x5110)) && !defined(__cplusplus)
-#elif (LZO_CC_GNUC >= 0x030400ul) && !(LZO_CC_PCC_GNUC) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-# define __lzo_struct_packed(s) struct s {
-# define __lzo_struct_packed_end() } __attribute__((__gcc_struct__,__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__gcc_struct__,__packed__));
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || (LZO_CC_PGI >= 0x0d0a00ul) || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_struct_packed(s) struct s {
-# define __lzo_struct_packed_end() } __attribute__((__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_struct_packed(s) __lzo_gnuc_extension__ struct s {
-# define __lzo_struct_packed_end() } __attribute__((__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_struct_packed(s) __pragma(pack(push,1)) struct s {
-# define __lzo_struct_packed_end() } __pragma(pack(pop));
-#elif (LZO_CC_WATCOMC && (__WATCOMC__ >= 900))
-# define __lzo_struct_packed(s) _Packed struct s {
-# define __lzo_struct_packed_end() };
-#endif
-#endif
-#if defined(__lzo_struct_packed) && !defined(__lzo_struct_packed_ma)
-# define __lzo_struct_packed_ma(s) __lzo_struct_packed(s)
-#endif
-#if defined(__lzo_struct_packed_end) && !defined(__lzo_struct_packed_ma_end)
-# define __lzo_struct_packed_ma_end() __lzo_struct_packed_end()
-#endif
-#if !defined(__lzo_byte_struct)
-#if defined(__lzo_struct_packed)
-# define __lzo_byte_struct(s,n) __lzo_struct_packed(s) unsigned char a[n]; __lzo_struct_packed_end()
-# define __lzo_byte_struct_ma(s,n) __lzo_struct_packed_ma(s) unsigned char a[n]; __lzo_struct_packed_ma_end()
-#elif (LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_PGI || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_byte_struct(s,n) struct s { unsigned char a[n]; } __attribute__((__packed__));
-# define __lzo_byte_struct_ma(s,n) struct s { unsigned char a[n]; } __lzo_may_alias __attribute__((__packed__));
-#endif
-#endif
-#if defined(__lzo_byte_struct) && !defined(__lzo_byte_struct_ma)
-# define __lzo_byte_struct_ma(s,n) __lzo_byte_struct(s,n)
-#endif
-#if !defined(__lzo_struct_align16) && (__lzo_HAVE_alignof)
-#if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x030000ul))
-#elif (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_CILLY || LZO_CC_PCC)
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_struct_align16(s) struct __declspec(align(16)) s {
-# define __lzo_struct_align16_end() };
-# define __lzo_struct_align32(s) struct __declspec(align(32)) s {
-# define __lzo_struct_align32_end() };
-# define __lzo_struct_align64(s) struct __declspec(align(64)) s {
-# define __lzo_struct_align64_end() };
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || (LZO_CC_IBMC >= 700) || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_struct_align16(s) struct s {
-# define __lzo_struct_align16_end() } __attribute__((__aligned__(16)));
-# define __lzo_struct_align32(s) struct s {
-# define __lzo_struct_align32_end() } __attribute__((__aligned__(32)));
-# define __lzo_struct_align64(s) struct s {
-# define __lzo_struct_align64_end() } __attribute__((__aligned__(64)));
-#endif
-#endif
-#if !defined(__lzo_union_um)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020700ul))
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER < 810))
-#elif (LZO_CC_PCC && (LZO_CC_PCC < 0x010100ul))
-#elif (LZO_CC_SUNPROC && (LZO_CC_SUNPROC < 0x5110)) && !defined(__cplusplus)
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || (LZO_CC_PGI >= 0x0d0a00ul) || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_union_am(s) union s {
-# define __lzo_union_am_end() } __lzo_may_alias;
-# define __lzo_union_um(s) union s {
-# define __lzo_union_um_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_union_am(s) __lzo_gnuc_extension__ union s {
-# define __lzo_union_am_end() } __lzo_may_alias;
-# define __lzo_union_um(s) __lzo_gnuc_extension__ union s {
-# define __lzo_union_um_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_union_um(s) __pragma(pack(push,1)) union s {
-# define __lzo_union_um_end() } __pragma(pack(pop));
-#elif (LZO_CC_WATCOMC && (__WATCOMC__ >= 900))
-# define __lzo_union_um(s) _Packed union s {
-# define __lzo_union_um_end() };
-#endif
-#endif
-#if !defined(__lzo_union_am)
-# define __lzo_union_am(s) union s {
-# define __lzo_union_am_end() };
-#endif
-#if !defined(__lzo_constructor)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_constructor __attribute__((__constructor__,__used__))
-#elif (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_constructor __attribute__((__constructor__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_constructor __attribute__((__constructor__,__used__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_constructor __attribute__((__constructor__))
-#endif
-#endif
-#if defined(__lzo_constructor)
-# ifndef __lzo_HAVE_constructor
-# define __lzo_HAVE_constructor 1
-# endif
-#endif
-#if !defined(__lzo_destructor)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_destructor __attribute__((__destructor__,__used__))
-#elif (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_destructor __attribute__((__destructor__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_destructor __attribute__((__destructor__,__used__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_destructor __attribute__((__destructor__))
-#endif
-#endif
-#if defined(__lzo_destructor)
-# ifndef __lzo_HAVE_destructor
-# define __lzo_HAVE_destructor 1
-# endif
-#endif
-#if (__lzo_HAVE_destructor) && !(__lzo_HAVE_constructor)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if !defined(__lzo_likely) && !defined(__lzo_unlikely)
-#if (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_IBMC >= 1010)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER >= 800))
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#endif
-#endif
-#if defined(__lzo_likely)
-# ifndef __lzo_HAVE_likely
-# define __lzo_HAVE_likely 1
-# endif
-#else
-# define __lzo_likely(e) (e)
-#endif
-#if defined(__lzo_very_likely)
-# ifndef __lzo_HAVE_very_likely
-# define __lzo_HAVE_very_likely 1
-# endif
-#else
-# define __lzo_very_likely(e) __lzo_likely(e)
-#endif
-#if defined(__lzo_unlikely)
-# ifndef __lzo_HAVE_unlikely
-# define __lzo_HAVE_unlikely 1
-# endif
-#else
-# define __lzo_unlikely(e) (e)
-#endif
-#if defined(__lzo_very_unlikely)
-# ifndef __lzo_HAVE_very_unlikely
-# define __lzo_HAVE_very_unlikely 1
-# endif
-#else
-# define __lzo_very_unlikely(e) __lzo_unlikely(e)
-#endif
-#if !defined(__lzo_loop_forever)
-# if (LZO_CC_IBMC)
-# define __lzo_loop_forever() LZO_BLOCK_BEGIN for (;;) { ; } LZO_BLOCK_END
-# else
-# define __lzo_loop_forever() do { ; } while __lzo_cte(1)
-# endif
-#endif
-#if !defined(__lzo_unreachable)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x020800ul)) && lzo_has_builtin(__builtin_unreachable)
-# define __lzo_unreachable() __builtin_unreachable();
-#elif (LZO_CC_GNUC >= 0x040500ul)
-# define __lzo_unreachable() __builtin_unreachable();
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 1300)) && 1
-# define __lzo_unreachable() __builtin_unreachable();
-#endif
-#endif
-#if defined(__lzo_unreachable)
-# ifndef __lzo_HAVE_unreachable
-# define __lzo_HAVE_unreachable 1
-# endif
-#else
-# if 0
-# define __lzo_unreachable() ((void)0);
-# else
-# define __lzo_unreachable() __lzo_loop_forever();
-# endif
-#endif
-#if !defined(lzo_unused_funcs_impl)
-# if 1 && (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || (LZO_CC_GNUC >= 0x020700ul) || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define lzo_unused_funcs_impl(r,f) static r __attribute__((__unused__)) f
-# elif 1 && (LZO_CC_BORLANDC || LZO_CC_GNUC)
-# define lzo_unused_funcs_impl(r,f) static r f
-# else
-# define lzo_unused_funcs_impl(r,f) __lzo_static_forceinline r f
-# endif
-#endif
-#ifndef __LZO_CTA_NAME
-#if (LZO_CFG_USE_COUNTER)
-# define __LZO_CTA_NAME(a) LZO_PP_ECONCAT2(a,__COUNTER__)
-#else
-# define __LZO_CTA_NAME(a) LZO_PP_ECONCAT2(a,__LINE__)
-#endif
-#endif
-#if !defined(LZO_COMPILE_TIME_ASSERT_HEADER)
-# if (LZO_CC_AZTECC || LZO_CC_ZORTECHC)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_DMC || LZO_CC_SYMANTECC)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1u-2*!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_TURBOC && (__TURBOC__ == 0x0295))
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020900ul)) && defined(__cplusplus)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN int __LZO_CTA_NAME(lzo_cta_f__)(int [1-2*!(e)]); LZO_EXTERN_C_END
-# elif (LZO_CC_GNUC) && defined(__CHECKER__) && defined(__SPARSE_CHECKER__)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN enum {__LZO_CTA_NAME(lzo_cta_e__)=1/!!(e)} __attribute__((__unused__)); LZO_EXTERN_C_END
-# else
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-2*!(e)]; LZO_EXTERN_C_END
-# endif
-#endif
-#if !defined(LZO_COMPILE_TIME_ASSERT)
-# if (LZO_CC_AZTECC)
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-!(e)];}
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x030000ul))
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)] __attribute__((__unused__));}
-# elif (LZO_CC_DMC || LZO_CC_PACIFICC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# elif (LZO_CC_GNUC) && defined(__CHECKER__) && defined(__SPARSE_CHECKER__)
-# define LZO_COMPILE_TIME_ASSERT(e) {(void) (0/!!(e));}
-# elif (LZO_CC_GNUC >= 0x040700ul) && (LZO_CFG_USE_COUNTER) && defined(__cplusplus)
-# define LZO_COMPILE_TIME_ASSERT(e) {enum {__LZO_CTA_NAME(lzo_cta_e__)=1/!!(e)} __attribute__((__unused__));}
-# elif (LZO_CC_GNUC >= 0x040700ul)
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)] __attribute__((__unused__));}
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# elif (LZO_CC_TURBOC && (__TURBOC__ == 0x0295))
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# else
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)];}
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(1 == 1)
-#if defined(__cplusplus)
-extern "C" { LZO_COMPILE_TIME_ASSERT_HEADER(2 == 2) }
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(3 == 3)
-#if (LZO_ARCH_I086 || LZO_ARCH_I386) && (LZO_OS_DOS16 || LZO_OS_DOS32 || LZO_OS_OS2 || LZO_OS_OS216 || LZO_OS_WIN16 || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (LZO_CC_GNUC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PACIFICC)
-# elif (LZO_CC_DMC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
-# define __lzo_cdecl __cdecl
-# define __lzo_cdecl_atexit /*empty*/
-# define __lzo_cdecl_main __cdecl
-# if (LZO_OS_OS2 && (LZO_CC_DMC || LZO_CC_SYMANTECC))
-# define __lzo_cdecl_qsort __pascal
-# elif (LZO_OS_OS2 && (LZO_CC_ZORTECHC))
-# define __lzo_cdecl_qsort _stdcall
-# else
-# define __lzo_cdecl_qsort __cdecl
-# endif
-# elif (LZO_CC_WATCOMC)
-# define __lzo_cdecl __cdecl
-# else
-# define __lzo_cdecl __cdecl
-# define __lzo_cdecl_atexit __cdecl
-# define __lzo_cdecl_main __cdecl
-# define __lzo_cdecl_qsort __cdecl
-# endif
-# if (LZO_CC_GNUC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PACIFICC || LZO_CC_WATCOMC)
-# elif (LZO_OS_OS2 && (LZO_CC_DMC || LZO_CC_SYMANTECC))
-# define __lzo_cdecl_sighandler __pascal
-# elif (LZO_OS_OS2 && (LZO_CC_ZORTECHC))
-# define __lzo_cdecl_sighandler _stdcall
-# elif (LZO_CC_MSC && (_MSC_VER >= 1400)) && defined(_M_CEE_PURE)
-# define __lzo_cdecl_sighandler __clrcall
-# elif (LZO_CC_MSC && (_MSC_VER >= 600 && _MSC_VER < 700))
-# if defined(_DLL)
-# define __lzo_cdecl_sighandler _far _cdecl _loadds
-# elif defined(_MT)
-# define __lzo_cdecl_sighandler _far _cdecl
-# else
-# define __lzo_cdecl_sighandler _cdecl
-# endif
-# else
-# define __lzo_cdecl_sighandler __cdecl
-# endif
-#elif (LZO_ARCH_I386) && (LZO_CC_WATCOMC)
-# define __lzo_cdecl __cdecl
-#elif (LZO_ARCH_M68K && LZO_OS_TOS && (LZO_CC_PUREC || LZO_CC_TURBOC))
-# define __lzo_cdecl cdecl
-#endif
-#if !defined(__lzo_cdecl)
-# define __lzo_cdecl /*empty*/
-#endif
-#if !defined(__lzo_cdecl_atexit)
-# define __lzo_cdecl_atexit /*empty*/
-#endif
-#if !defined(__lzo_cdecl_main)
-# define __lzo_cdecl_main /*empty*/
-#endif
-#if !defined(__lzo_cdecl_qsort)
-# define __lzo_cdecl_qsort /*empty*/
-#endif
-#if !defined(__lzo_cdecl_sighandler)
-# define __lzo_cdecl_sighandler /*empty*/
-#endif
-#if !defined(__lzo_cdecl_va)
-# define __lzo_cdecl_va __lzo_cdecl
-#endif
-#if !(LZO_CFG_NO_WINDOWS_H)
-#if !defined(LZO_HAVE_WINDOWS_H)
-#if (LZO_OS_CYGWIN || (LZO_OS_EMX && defined(__RSXNT__)) || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (LZO_CC_WATCOMC && (__WATCOMC__ < 1000))
-# elif ((LZO_OS_WIN32 && defined(__PW32__)) && (LZO_CC_GNUC && (LZO_CC_GNUC < 0x030000ul)))
-# elif ((LZO_OS_CYGWIN || defined(__MINGW32__)) && (LZO_CC_GNUC && (LZO_CC_GNUC < 0x025f00ul)))
-# else
-# define LZO_HAVE_WINDOWS_H 1
-# endif
-#endif
-#endif
-#endif
-#ifndef LZO_SIZEOF_SHORT
-#if defined(SIZEOF_SHORT)
-# define LZO_SIZEOF_SHORT (SIZEOF_SHORT)
-#elif defined(__SIZEOF_SHORT__)
-# define LZO_SIZEOF_SHORT (__SIZEOF_SHORT__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_INT
-#if defined(SIZEOF_INT)
-# define LZO_SIZEOF_INT (SIZEOF_INT)
-#elif defined(__SIZEOF_INT__)
-# define LZO_SIZEOF_INT (__SIZEOF_INT__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_LONG
-#if defined(SIZEOF_LONG)
-# define LZO_SIZEOF_LONG (SIZEOF_LONG)
-#elif defined(__SIZEOF_LONG__)
-# define LZO_SIZEOF_LONG (__SIZEOF_LONG__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_LONG_LONG
-#if defined(SIZEOF_LONG_LONG)
-# define LZO_SIZEOF_LONG_LONG (SIZEOF_LONG_LONG)
-#elif defined(__SIZEOF_LONG_LONG__)
-# define LZO_SIZEOF_LONG_LONG (__SIZEOF_LONG_LONG__)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT16
-#if defined(SIZEOF___INT16)
-# define LZO_SIZEOF___INT16 (SIZEOF___INT16)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT32
-#if defined(SIZEOF___INT32)
-# define LZO_SIZEOF___INT32 (SIZEOF___INT32)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT64
-#if defined(SIZEOF___INT64)
-# define LZO_SIZEOF___INT64 (SIZEOF___INT64)
-#endif
-#endif
-#ifndef LZO_SIZEOF_VOID_P
-#if defined(SIZEOF_VOID_P)
-# define LZO_SIZEOF_VOID_P (SIZEOF_VOID_P)
-#elif defined(__SIZEOF_POINTER__)
-# define LZO_SIZEOF_VOID_P (__SIZEOF_POINTER__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_SIZE_T
-#if defined(SIZEOF_SIZE_T)
-# define LZO_SIZEOF_SIZE_T (SIZEOF_SIZE_T)
-#elif defined(__SIZEOF_SIZE_T__)
-# define LZO_SIZEOF_SIZE_T (__SIZEOF_SIZE_T__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_PTRDIFF_T
-#if defined(SIZEOF_PTRDIFF_T)
-# define LZO_SIZEOF_PTRDIFF_T (SIZEOF_PTRDIFF_T)
-#elif defined(__SIZEOF_PTRDIFF_T__)
-# define LZO_SIZEOF_PTRDIFF_T (__SIZEOF_PTRDIFF_T__)
-#endif
-#endif
-#define __LZO_LSR(x,b) (((x)+0ul) >> (b))
-#if !defined(LZO_SIZEOF_SHORT)
-# if (LZO_ARCH_CRAY_PVP)
-# define LZO_SIZEOF_SHORT 8
-# elif (USHRT_MAX == LZO_0xffffL)
-# define LZO_SIZEOF_SHORT 2
-# elif (__LZO_LSR(USHRT_MAX,7) == 1)
-# define LZO_SIZEOF_SHORT 1
-# elif (__LZO_LSR(USHRT_MAX,15) == 1)
-# define LZO_SIZEOF_SHORT 2
-# elif (__LZO_LSR(USHRT_MAX,31) == 1)
-# define LZO_SIZEOF_SHORT 4
-# elif (__LZO_LSR(USHRT_MAX,63) == 1)
-# define LZO_SIZEOF_SHORT 8
-# elif (__LZO_LSR(USHRT_MAX,127) == 1)
-# define LZO_SIZEOF_SHORT 16
-# else
-# error "LZO_SIZEOF_SHORT"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_SHORT == sizeof(short))
-#if !defined(LZO_SIZEOF_INT)
-# if (LZO_ARCH_CRAY_PVP)
-# define LZO_SIZEOF_INT 8
-# elif (UINT_MAX == LZO_0xffffL)
-# define LZO_SIZEOF_INT 2
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_SIZEOF_INT 4
-# elif (__LZO_LSR(UINT_MAX,7) == 1)
-# define LZO_SIZEOF_INT 1
-# elif (__LZO_LSR(UINT_MAX,15) == 1)
-# define LZO_SIZEOF_INT 2
-# elif (__LZO_LSR(UINT_MAX,31) == 1)
-# define LZO_SIZEOF_INT 4
-# elif (__LZO_LSR(UINT_MAX,63) == 1)
-# define LZO_SIZEOF_INT 8
-# elif (__LZO_LSR(UINT_MAX,127) == 1)
-# define LZO_SIZEOF_INT 16
-# else
-# error "LZO_SIZEOF_INT"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_INT == sizeof(int))
-#if !defined(LZO_SIZEOF_LONG)
-# if (ULONG_MAX == LZO_0xffffffffL)
-# define LZO_SIZEOF_LONG 4
-# elif (__LZO_LSR(ULONG_MAX,7) == 1)
-# define LZO_SIZEOF_LONG 1
-# elif (__LZO_LSR(ULONG_MAX,15) == 1)
-# define LZO_SIZEOF_LONG 2
-# elif (__LZO_LSR(ULONG_MAX,31) == 1)
-# define LZO_SIZEOF_LONG 4
-# elif (__LZO_LSR(ULONG_MAX,39) == 1)
-# define LZO_SIZEOF_LONG 5
-# elif (__LZO_LSR(ULONG_MAX,63) == 1)
-# define LZO_SIZEOF_LONG 8
-# elif (__LZO_LSR(ULONG_MAX,127) == 1)
-# define LZO_SIZEOF_LONG 16
-# else
-# error "LZO_SIZEOF_LONG"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_LONG == sizeof(long))
-#if !defined(LZO_SIZEOF_LONG_LONG) && !defined(LZO_SIZEOF___INT64)
-#if (LZO_SIZEOF_LONG > 0 && LZO_SIZEOF_LONG < 8)
-# if defined(__LONG_MAX__) && defined(__LONG_LONG_MAX__)
-# if (LZO_CC_GNUC >= 0x030300ul)
-# if ((__LONG_MAX__-0) == (__LONG_LONG_MAX__-0))
-# define LZO_SIZEOF_LONG_LONG LZO_SIZEOF_LONG
-# elif (__LZO_LSR(__LONG_LONG_MAX__,30) == 1)
-# define LZO_SIZEOF_LONG_LONG 4
-# endif
-# endif
-# endif
-#endif
-#endif
-#if !defined(LZO_SIZEOF_LONG_LONG) && !defined(LZO_SIZEOF___INT64)
-#if (LZO_SIZEOF_LONG > 0 && LZO_SIZEOF_LONG < 8)
-#if (LZO_ARCH_I086 && LZO_CC_DMC)
-#elif (LZO_CC_CILLY) && defined(__GNUC__)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif ((LZO_OS_WIN32 || LZO_OS_WIN64 || defined(_WIN32)) && LZO_CC_MSC && (_MSC_VER >= 1400))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_OS_WIN64 || defined(_WIN64))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_DMC))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_SYMANTECC && (__SC__ >= 0x700)))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_INTELC && defined(__linux__)))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_MWERKS || LZO_CC_PELLESC || LZO_CC_PGI || LZO_CC_SUNPROC))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_INTELC || LZO_CC_MSC))
-# define LZO_SIZEOF___INT64 8
-#elif ((LZO_OS_WIN32 || defined(_WIN32)) && (LZO_CC_MSC))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0520)))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_WATCOMC && (__WATCOMC__ >= 1100)))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_CC_GHS && defined(__LLONG_BIT) && ((__LLONG_BIT-0) == 64))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_WATCOMC && defined(_INTEGRAL_MAX_BITS) && ((_INTEGRAL_MAX_BITS-0) == 64))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_OS_OS400 || defined(__OS400__)) && defined(__LLP64_IFC__)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (defined(__vms) || defined(__VMS)) && ((__INITIAL_POINTER_SIZE-0) == 64)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_SDCC) && (LZO_SIZEOF_INT == 2)
-#elif 1 && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
-# define LZO_SIZEOF_LONG_LONG 8
-#endif
-#endif
-#endif
-#if defined(__cplusplus) && (LZO_CC_GNUC)
-# if (LZO_CC_GNUC < 0x020800ul)
-# undef LZO_SIZEOF_LONG_LONG
-# endif
-#endif
-#if (LZO_CFG_NO_LONG_LONG)
-# undef LZO_SIZEOF_LONG_LONG
-#elif defined(__NO_LONG_LONG)
-# undef LZO_SIZEOF_LONG_LONG
-#elif defined(_NO_LONGLONG)
-# undef LZO_SIZEOF_LONG_LONG
-#endif
-#if !defined(LZO_WORDSIZE)
-#if (LZO_ARCH_ALPHA)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_AMD64)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_AVR)
-# define LZO_WORDSIZE 1
-#elif (LZO_ARCH_H8300)
-# if defined(__NORMAL_MODE__)
-# define LZO_WORDSIZE 4
-# elif defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_WORDSIZE 4
-# else
-# define LZO_WORDSIZE 2
-# endif
-#elif (LZO_ARCH_I086)
-# define LZO_WORDSIZE 2
-#elif (LZO_ARCH_IA64)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_M16C)
-# define LZO_WORDSIZE 2
-#elif (LZO_ARCH_SPU)
-# define LZO_WORDSIZE 4
-#elif (LZO_ARCH_Z80)
-# define LZO_WORDSIZE 1
-#elif (LZO_SIZEOF_LONG == 8) && ((defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__))
-# define LZO_WORDSIZE 8
-#elif (LZO_OS_OS400 || defined(__OS400__))
-# define LZO_WORDSIZE 8
-#elif (defined(__vms) || defined(__VMS)) && (__INITIAL_POINTER_SIZE+0 == 64)
-# define LZO_WORDSIZE 8
-#endif
-#endif
-#if !defined(LZO_SIZEOF_VOID_P)
-#if defined(__ILP32__) || defined(__ILP32) || defined(_ILP32)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(int) == 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 4)
-# define LZO_SIZEOF_VOID_P 4
-#elif defined(__ILP64__) || defined(__ILP64) || defined(_ILP64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(int) == 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 8)
-# define LZO_SIZEOF_VOID_P 8
-#elif defined(__LLP64__) || defined(__LLP64) || defined(_LLP64) || defined(_WIN64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 4)
-# define LZO_SIZEOF_VOID_P 8
-#elif defined(__LP64__) || defined(__LP64) || defined(_LP64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 8)
-# define LZO_SIZEOF_VOID_P 8
-#elif (LZO_ARCH_AVR)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_ARCH_C166 || LZO_ARCH_MCS51 || LZO_ARCH_MCS251 || LZO_ARCH_MSP430)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_ARCH_H8300)
-# if defined(__NORMAL_MODE__)
-# define LZO_SIZEOF_VOID_P 2
-# elif defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# define LZO_SIZEOF_VOID_P 2
-# endif
-# if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x040000ul)) && (LZO_SIZEOF_INT == 4)
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_INT
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_INT
-# endif
-#elif (LZO_ARCH_I086)
-# if (LZO_MM_TINY || LZO_MM_SMALL || LZO_MM_MEDIUM)
-# define LZO_SIZEOF_VOID_P 2
-# elif (LZO_MM_COMPACT || LZO_MM_LARGE || LZO_MM_HUGE)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# error "invalid LZO_ARCH_I086 memory model"
-# endif
-#elif (LZO_ARCH_M16C)
-# if defined(__m32c_cpu__) || defined(__m32cm_cpu__)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# define LZO_SIZEOF_VOID_P 2
-# endif
-#elif (LZO_ARCH_SPU)
-# define LZO_SIZEOF_VOID_P 4
-#elif (LZO_ARCH_Z80)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_SIZEOF_LONG == 8) && ((defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__))
-# define LZO_SIZEOF_VOID_P 4
-#elif (LZO_OS_OS400 || defined(__OS400__))
-# if defined(__LLP64_IFC__)
-# define LZO_SIZEOF_VOID_P 8
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-# else
-# define LZO_SIZEOF_VOID_P 16
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-# endif
-#elif (defined(__vms) || defined(__VMS)) && (__INITIAL_POINTER_SIZE+0 == 64)
-# define LZO_SIZEOF_VOID_P 8
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-#endif
-#endif
-#if !defined(LZO_SIZEOF_VOID_P)
-# define LZO_SIZEOF_VOID_P LZO_SIZEOF_LONG
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_VOID_P == sizeof(void *))
-#if !defined(LZO_SIZEOF_SIZE_T)
-#if (LZO_ARCH_I086 || LZO_ARCH_M16C)
-# define LZO_SIZEOF_SIZE_T 2
-#endif
-#endif
-#if !defined(LZO_SIZEOF_SIZE_T)
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_VOID_P
-#endif
-#if defined(offsetof)
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_SIZE_T == sizeof(size_t))
-#endif
-#if !defined(LZO_SIZEOF_PTRDIFF_T)
-#if (LZO_ARCH_I086)
-# if (LZO_MM_TINY || LZO_MM_SMALL || LZO_MM_MEDIUM || LZO_MM_HUGE)
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_VOID_P
-# elif (LZO_MM_COMPACT || LZO_MM_LARGE)
-# if (LZO_CC_BORLANDC || LZO_CC_TURBOC)
-# define LZO_SIZEOF_PTRDIFF_T 4
-# else
-# define LZO_SIZEOF_PTRDIFF_T 2
-# endif
-# else
-# error "invalid LZO_ARCH_I086 memory model"
-# endif
-#endif
-#endif
-#if !defined(LZO_SIZEOF_PTRDIFF_T)
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_SIZE_T
-#endif
-#if defined(offsetof)
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_PTRDIFF_T == sizeof(ptrdiff_t))
-#endif
-#if !defined(LZO_WORDSIZE)
-# define LZO_WORDSIZE LZO_SIZEOF_VOID_P
-#endif
-#if (LZO_ABI_NEUTRAL_ENDIAN)
-# undef LZO_ABI_BIG_ENDIAN
-# undef LZO_ABI_LITTLE_ENDIAN
-#elif !(LZO_ABI_BIG_ENDIAN) && !(LZO_ABI_LITTLE_ENDIAN)
-#if (LZO_ARCH_ALPHA) && (LZO_ARCH_CRAY_MPP)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif (LZO_ARCH_IA64) && (LZO_OS_POSIX_LINUX || LZO_OS_WIN64)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif (LZO_ARCH_ALPHA || LZO_ARCH_AMD64 || LZO_ARCH_BLACKFIN || LZO_ARCH_CRIS || LZO_ARCH_I086 || LZO_ARCH_I386 || LZO_ARCH_MSP430)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif (LZO_ARCH_AVR32 || LZO_ARCH_M68K || LZO_ARCH_S390 || LZO_ARCH_SPU)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && defined(__IAR_SYSTEMS_ICC__) && defined(__LITTLE_ENDIAN__)
-# if (__LITTLE_ENDIAN__ == 1)
-# define LZO_ABI_LITTLE_ENDIAN 1
-# else
-# define LZO_ABI_BIG_ENDIAN 1
-# endif
-#elif 1 && defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARM_BIG_ENDIAN) && ((__ARM_BIG_ENDIAN)+0)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARMEB__) && !defined(__ARMEL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARMEL__) && !defined(__ARMEB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM && LZO_CC_ARMCC_ARMCC)
-# if defined(__BIG_ENDIAN) && defined(__LITTLE_ENDIAN)
-# error "unexpected configuration - check your compiler defines"
-# elif defined(__BIG_ENDIAN)
-# define LZO_ABI_BIG_ENDIAN 1
-# else
-# define LZO_ABI_LITTLE_ENDIAN 1
-# endif
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__ARM_BIG_ENDIAN) && ((__ARM_BIG_ENDIAN)+0)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__AARCH64EB__) && !defined(__AARCH64EL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__AARCH64EL__) && !defined(__AARCH64EB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_MIPS) && defined(__MIPSEB__) && !defined(__MIPSEL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_MIPS) && defined(__MIPSEL__) && !defined(__MIPSEB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#endif
-#endif
-#if (LZO_ABI_BIG_ENDIAN) && (LZO_ABI_LITTLE_ENDIAN)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ABI_BIG_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "be"
-#elif (LZO_ABI_LITTLE_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "le"
-#elif (LZO_ABI_NEUTRAL_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "neutral"
-#endif
-#if (LZO_SIZEOF_INT == 1 && LZO_SIZEOF_LONG == 2 && LZO_SIZEOF_VOID_P == 2)
-# define LZO_ABI_I8LP16 1
-# define LZO_INFO_ABI_PM "i8lp16"
-#elif (LZO_SIZEOF_INT == 2 && LZO_SIZEOF_LONG == 2 && LZO_SIZEOF_VOID_P == 2)
-# define LZO_ABI_ILP16 1
-# define LZO_INFO_ABI_PM "ilp16"
-#elif (LZO_SIZEOF_INT == 2 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_LP32 1
-# define LZO_INFO_ABI_PM "lp32"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_ILP32 1
-# define LZO_INFO_ABI_PM "ilp32"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 8 && LZO_SIZEOF_SIZE_T == 8)
-# define LZO_ABI_LLP64 1
-# define LZO_INFO_ABI_PM "llp64"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 8)
-# define LZO_ABI_LP64 1
-# define LZO_INFO_ABI_PM "lp64"
-#elif (LZO_SIZEOF_INT == 8 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 8)
-# define LZO_ABI_ILP64 1
-# define LZO_INFO_ABI_PM "ilp64"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_IP32L64 1
-# define LZO_INFO_ABI_PM "ip32l64"
-#endif
-#if 0
-#elif !defined(__LZO_LIBC_OVERRIDE)
-#if (LZO_LIBC_NAKED)
-# define LZO_INFO_LIBC "naked"
-#elif (LZO_LIBC_FREESTANDING)
-# define LZO_INFO_LIBC "freestanding"
-#elif (LZO_LIBC_MOSTLY_FREESTANDING)
-# define LZO_INFO_LIBC "mfreestanding"
-#elif (LZO_LIBC_ISOC90)
-# define LZO_INFO_LIBC "isoc90"
-#elif (LZO_LIBC_ISOC99)
-# define LZO_INFO_LIBC "isoc99"
-#elif (LZO_CC_ARMCC_ARMCC) && defined(__ARMCLIB_VERSION)
-# define LZO_LIBC_ISOC90 1
-# define LZO_INFO_LIBC "isoc90"
-#elif defined(__dietlibc__)
-# define LZO_LIBC_DIETLIBC 1
-# define LZO_INFO_LIBC "dietlibc"
-#elif defined(_NEWLIB_VERSION)
-# define LZO_LIBC_NEWLIB 1
-# define LZO_INFO_LIBC "newlib"
-#elif defined(__UCLIBC__) && defined(__UCLIBC_MAJOR__) && defined(__UCLIBC_MINOR__)
-# if defined(__UCLIBC_SUBLEVEL__)
-# define LZO_LIBC_UCLIBC (__UCLIBC_MAJOR__ * 0x10000L + (__UCLIBC_MINOR__-0) * 0x100 + (__UCLIBC_SUBLEVEL__-0))
-# else
-# define LZO_LIBC_UCLIBC 0x00090bL
-# endif
-# define LZO_INFO_LIBC "uc" "libc"
-#elif defined(__GLIBC__) && defined(__GLIBC_MINOR__)
-# define LZO_LIBC_GLIBC (__GLIBC__ * 0x10000L + (__GLIBC_MINOR__-0) * 0x100)
-# define LZO_INFO_LIBC "glibc"
-#elif (LZO_CC_MWERKS) && defined(__MSL__)
-# define LZO_LIBC_MSL __MSL__
-# define LZO_INFO_LIBC "msl"
-#elif 1 && defined(__IAR_SYSTEMS_ICC__)
-# define LZO_LIBC_ISOC90 1
-# define LZO_INFO_LIBC "isoc90"
-#else
-# define LZO_LIBC_DEFAULT 1
-# define LZO_INFO_LIBC "default"
-#endif
-#endif
-#if (LZO_ARCH_I386 && (LZO_OS_DOS32 || LZO_OS_WIN32) && (LZO_CC_DMC || LZO_CC_INTELC || LZO_CC_MSC || LZO_CC_PELLESC))
-# define LZO_ASM_SYNTAX_MSC 1
-#elif (LZO_OS_WIN64 && (LZO_CC_DMC || LZO_CC_INTELC || LZO_CC_MSC || LZO_CC_PELLESC))
-#elif (LZO_ARCH_I386 && LZO_CC_GNUC && (LZO_CC_GNUC == 0x011f00ul))
-#elif (LZO_ARCH_I386 && (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC || LZO_CC_PATHSCALE))
-# define LZO_ASM_SYNTAX_GNUC 1
-#elif (LZO_ARCH_AMD64 && (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC || LZO_CC_PATHSCALE))
-# define LZO_ASM_SYNTAX_GNUC 1
-#elif (LZO_CC_GNUC)
-# define LZO_ASM_SYNTAX_GNUC 1
-#endif
-#if (LZO_ASM_SYNTAX_GNUC)
-#if (LZO_ARCH_I386 && LZO_CC_GNUC && (LZO_CC_GNUC < 0x020000ul))
-# define __LZO_ASM_CLOBBER "ax"
-# define __LZO_ASM_CLOBBER_LIST_CC /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER < 1000))
-# define __LZO_ASM_CLOBBER "memory"
-# define __LZO_ASM_CLOBBER_LIST_CC /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY : "memory"
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#else
-# define __LZO_ASM_CLOBBER "cc", "memory"
-# define __LZO_ASM_CLOBBER_LIST_CC : "cc"
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY : "cc", "memory"
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#endif
-#endif
-#if (LZO_ARCH_ALPHA)
-# define LZO_OPT_AVOID_UINT_INDEX 1
-#elif (LZO_ARCH_AMD64)
-# define LZO_OPT_AVOID_INT_INDEX 1
-# define LZO_OPT_AVOID_UINT_INDEX 1
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-#elif (LZO_ARCH_ARM)
-# if defined(__ARM_FEATURE_UNALIGNED)
-# if ((__ARM_FEATURE_UNALIGNED)+0)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-# elif 1 && (LZO_ARCH_ARM_THUMB2)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# elif 1 && defined(__TARGET_ARCH_ARM) && ((__TARGET_ARCH_ARM)+0 >= 7)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# elif 1 && defined(__TARGET_ARCH_ARM) && ((__TARGET_ARCH_ARM)+0 >= 6) && (defined(__TARGET_PROFILE_A) || defined(__TARGET_PROFILE_R))
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-#elif (LZO_ARCH_ARM64)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-#elif (LZO_ARCH_CRIS)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-#elif (LZO_ARCH_I386)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-#elif (LZO_ARCH_IA64)
-# define LZO_OPT_AVOID_INT_INDEX 1
-# define LZO_OPT_AVOID_UINT_INDEX 1
-# define LZO_OPT_PREFER_POSTINC 1
-#elif (LZO_ARCH_M68K)
-# define LZO_OPT_PREFER_POSTINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-# if defined(__mc68020__) && !defined(__mcoldfire__)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-#elif (LZO_ARCH_MIPS)
-# define LZO_OPT_AVOID_UINT_INDEX 1
-#elif (LZO_ARCH_POWERPC)
-# define LZO_OPT_PREFER_PREINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-# if (LZO_ABI_BIG_ENDIAN)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# if (LZO_WORDSIZE == 8)
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-# endif
-# endif
-#elif (LZO_ARCH_S390)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# if (LZO_WORDSIZE == 8)
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-# endif
-#elif (LZO_ARCH_SH)
-# define LZO_OPT_PREFER_POSTINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-#endif
-#ifndef LZO_CFG_NO_INLINE_ASM
-#if (LZO_ABI_NEUTRAL_ENDIAN) || (LZO_ARCH_GENERIC)
-# define LZO_CFG_NO_INLINE_ASM 1
-#elif (LZO_CC_LLVM)
-# define LZO_CFG_NO_INLINE_ASM 1
-#endif
-#endif
-#if (LZO_CFG_NO_INLINE_ASM)
-# undef LZO_ASM_SYNTAX_MSC
-# undef LZO_ASM_SYNTAX_GNUC
-# undef __LZO_ASM_CLOBBER
-# undef __LZO_ASM_CLOBBER_LIST_CC
-# undef __LZO_ASM_CLOBBER_LIST_CC_MEMORY
-# undef __LZO_ASM_CLOBBER_LIST_EMPTY
-#endif
-#ifndef LZO_CFG_NO_UNALIGNED
-#if (LZO_ABI_NEUTRAL_ENDIAN) || (LZO_ARCH_GENERIC)
-# define LZO_CFG_NO_UNALIGNED 1
-#endif
-#endif
-#if (LZO_CFG_NO_UNALIGNED)
-# undef LZO_OPT_UNALIGNED16
-# undef LZO_OPT_UNALIGNED32
-# undef LZO_OPT_UNALIGNED64
-#endif
-#if defined(__LZO_INFOSTR_MM)
-#elif (LZO_MM_FLAT) && (defined(__LZO_INFOSTR_PM) || defined(LZO_INFO_ABI_PM))
-# define __LZO_INFOSTR_MM ""
-#elif defined(LZO_INFO_MM)
-# define __LZO_INFOSTR_MM "." LZO_INFO_MM
-#else
-# define __LZO_INFOSTR_MM ""
-#endif
-#if defined(__LZO_INFOSTR_PM)
-#elif defined(LZO_INFO_ABI_PM)
-# define __LZO_INFOSTR_PM "." LZO_INFO_ABI_PM
-#else
-# define __LZO_INFOSTR_PM ""
-#endif
-#if defined(__LZO_INFOSTR_ENDIAN)
-#elif defined(LZO_INFO_ABI_ENDIAN)
-# define __LZO_INFOSTR_ENDIAN "." LZO_INFO_ABI_ENDIAN
-#else
-# define __LZO_INFOSTR_ENDIAN ""
-#endif
-#if defined(__LZO_INFOSTR_OSNAME)
-#elif defined(LZO_INFO_OS_CONSOLE)
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS "." LZO_INFO_OS_CONSOLE
-#elif defined(LZO_INFO_OS_POSIX)
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS "." LZO_INFO_OS_POSIX
-#else
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS
-#endif
-#if defined(__LZO_INFOSTR_LIBC)
-#elif defined(LZO_INFO_LIBC)
-# define __LZO_INFOSTR_LIBC "." LZO_INFO_LIBC
-#else
-# define __LZO_INFOSTR_LIBC ""
-#endif
-#if defined(__LZO_INFOSTR_CCVER)
-#elif defined(LZO_INFO_CCVER)
-# define __LZO_INFOSTR_CCVER " " LZO_INFO_CCVER
-#else
-# define __LZO_INFOSTR_CCVER ""
-#endif
-#define LZO_INFO_STRING \
- LZO_INFO_ARCH __LZO_INFOSTR_MM __LZO_INFOSTR_PM __LZO_INFOSTR_ENDIAN \
- " " __LZO_INFOSTR_OSNAME __LZO_INFOSTR_LIBC " " LZO_INFO_CC __LZO_INFOSTR_CCVER
-#if !(LZO_CFG_SKIP_LZO_TYPES)
-#if (!(LZO_SIZEOF_SHORT+0 > 0 && LZO_SIZEOF_INT+0 > 0 && LZO_SIZEOF_LONG+0 > 0))
-# error "missing defines for sizes"
-#endif
-#if (!(LZO_SIZEOF_PTRDIFF_T+0 > 0 && LZO_SIZEOF_SIZE_T+0 > 0 && LZO_SIZEOF_VOID_P+0 > 0))
-# error "missing defines for sizes"
-#endif
-#define LZO_TYPEOF_CHAR 1u
-#define LZO_TYPEOF_SHORT 2u
-#define LZO_TYPEOF_INT 3u
-#define LZO_TYPEOF_LONG 4u
-#define LZO_TYPEOF_LONG_LONG 5u
-#define LZO_TYPEOF___INT8 17u
-#define LZO_TYPEOF___INT16 18u
-#define LZO_TYPEOF___INT32 19u
-#define LZO_TYPEOF___INT64 20u
-#define LZO_TYPEOF___INT128 21u
-#define LZO_TYPEOF___INT256 22u
-#define LZO_TYPEOF___MODE_QI 33u
-#define LZO_TYPEOF___MODE_HI 34u
-#define LZO_TYPEOF___MODE_SI 35u
-#define LZO_TYPEOF___MODE_DI 36u
-#define LZO_TYPEOF___MODE_TI 37u
-#define LZO_TYPEOF_CHAR_P 129u
-#if !defined(lzo_llong_t)
-#if (LZO_SIZEOF_LONG_LONG+0 > 0)
-__lzo_gnuc_extension__ typedef long long lzo_llong_t__;
-__lzo_gnuc_extension__ typedef unsigned long long lzo_ullong_t__;
-# define lzo_llong_t lzo_llong_t__
-# define lzo_ullong_t lzo_ullong_t__
-#endif
-#endif
-#if !defined(lzo_int16e_t)
-#if (LZO_SIZEOF_LONG == 2)
-# define lzo_int16e_t long
-# define lzo_uint16e_t unsigned long
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_INT == 2)
-# define lzo_int16e_t int
-# define lzo_uint16e_t unsigned int
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == 2)
-# define lzo_int16e_t short int
-# define lzo_uint16e_t unsigned short int
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_SHORT
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_HI) && (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x025f00ul) || LZO_CC_LLVM)
- typedef int lzo_int16e_hi_t__ __attribute__((__mode__(__HI__)));
- typedef unsigned int lzo_uint16e_hi_t__ __attribute__((__mode__(__HI__)));
-# define lzo_int16e_t lzo_int16e_hi_t__
-# define lzo_uint16e_t lzo_uint16e_hi_t__
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF___MODE_HI
-#elif (LZO_SIZEOF___INT16 == 2)
-# define lzo_int16e_t __int16
-# define lzo_uint16e_t unsigned __int16
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF___INT16
-#else
-#endif
-#endif
-#if defined(lzo_int16e_t)
-# define LZO_SIZEOF_LZO_INT16E_T 2
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16e_t) == 2)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16e_t) == LZO_SIZEOF_LZO_INT16E_T)
-#endif
-#if !defined(lzo_int32e_t)
-#if (LZO_SIZEOF_LONG == 4)
-# define lzo_int32e_t long int
-# define lzo_uint32e_t unsigned long int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_INT == 4)
-# define lzo_int32e_t int
-# define lzo_uint32e_t unsigned int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == 4)
-# define lzo_int32e_t short int
-# define lzo_uint32e_t unsigned short int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_SHORT
-#elif (LZO_SIZEOF_LONG_LONG == 4)
-# define lzo_int32e_t lzo_llong_t
-# define lzo_uint32e_t lzo_ullong_t
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_LONG_LONG
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_SI) && (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x025f00ul) || LZO_CC_LLVM) && (__INT_MAX__+0 > 2147483647L)
- typedef int lzo_int32e_si_t__ __attribute__((__mode__(__SI__)));
- typedef unsigned int lzo_uint32e_si_t__ __attribute__((__mode__(__SI__)));
-# define lzo_int32e_t lzo_int32e_si_t__
-# define lzo_uint32e_t lzo_uint32e_si_t__
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___MODE_SI
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_SI) && (LZO_CC_GNUC >= 0x025f00ul) && defined(__AVR__) && (__LONG_MAX__+0 == 32767L)
- typedef int lzo_int32e_si_t__ __attribute__((__mode__(__SI__)));
- typedef unsigned int lzo_uint32e_si_t__ __attribute__((__mode__(__SI__)));
-# define lzo_int32e_t lzo_int32e_si_t__
-# define lzo_uint32e_t lzo_uint32e_si_t__
-# define LZO_INT32_C(c) (c##LL)
-# define LZO_UINT32_C(c) (c##ULL)
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___MODE_SI
-#elif (LZO_SIZEOF___INT32 == 4)
-# define lzo_int32e_t __int32
-# define lzo_uint32e_t unsigned __int32
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___INT32
-#else
-#endif
-#endif
-#if defined(lzo_int32e_t)
-# define LZO_SIZEOF_LZO_INT32E_T 4
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32e_t) == 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32e_t) == LZO_SIZEOF_LZO_INT32E_T)
-#endif
-#if !defined(lzo_int64e_t)
-#if (LZO_SIZEOF___INT64 == 8)
-# if (LZO_CC_BORLANDC) && !(LZO_CFG_TYPE_PREFER___INT64)
-# define LZO_CFG_TYPE_PREFER___INT64 1
-# endif
-#endif
-#if (LZO_SIZEOF_INT == 8) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_int64e_t int
-# define lzo_uint64e_t unsigned int
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_LONG == 8)
-# define lzo_int64e_t long int
-# define lzo_uint64e_t unsigned long int
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_LONG_LONG == 8) && !(LZO_CFG_TYPE_PREFER___INT64)
-# define lzo_int64e_t lzo_llong_t
-# define lzo_uint64e_t lzo_ullong_t
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_LONG_LONG
-# if (LZO_CC_BORLANDC)
-# define LZO_INT64_C(c) ((c) + 0ll)
-# define LZO_UINT64_C(c) ((c) + 0ull)
-# elif 0
-# define LZO_INT64_C(c) (__lzo_gnuc_extension__ (c##LL))
-# define LZO_UINT64_C(c) (__lzo_gnuc_extension__ (c##ULL))
-# else
-# define LZO_INT64_C(c) (c##LL)
-# define LZO_UINT64_C(c) (c##ULL)
-# endif
-#elif (LZO_SIZEOF___INT64 == 8)
-# define lzo_int64e_t __int64
-# define lzo_uint64e_t unsigned __int64
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF___INT64
-# if (LZO_CC_BORLANDC)
-# define LZO_INT64_C(c) ((c) + 0i64)
-# define LZO_UINT64_C(c) ((c) + 0ui64)
-# else
-# define LZO_INT64_C(c) (c##i64)
-# define LZO_UINT64_C(c) (c##ui64)
-# endif
-#else
-#endif
-#endif
-#if defined(lzo_int64e_t)
-# define LZO_SIZEOF_LZO_INT64E_T 8
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64e_t) == 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64e_t) == LZO_SIZEOF_LZO_INT64E_T)
-#endif
-#if !defined(lzo_int32l_t)
-#if defined(lzo_int32e_t)
-# define lzo_int32l_t lzo_int32e_t
-# define lzo_uint32l_t lzo_uint32e_t
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_LZO_INT32E_T
-# define LZO_TYPEOF_LZO_INT32L_T LZO_TYPEOF_LZO_INT32E_T
-#elif (LZO_SIZEOF_INT >= 4) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_int32l_t int
-# define lzo_uint32l_t unsigned int
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INT32L_T LZO_SIZEOF_INT
-#elif (LZO_SIZEOF_LONG >= 4)
-# define lzo_int32l_t long int
-# define lzo_uint32l_t unsigned long int
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_INT32L_T LZO_SIZEOF_LONG
-#else
-# error "lzo_int32l_t"
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32l_t) >= 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32l_t) == LZO_SIZEOF_LZO_INT32L_T)
-#endif
-#if !defined(lzo_int64l_t)
-#if defined(lzo_int64e_t)
-# define lzo_int64l_t lzo_int64e_t
-# define lzo_uint64l_t lzo_uint64e_t
-# define LZO_SIZEOF_LZO_INT64L_T LZO_SIZEOF_LZO_INT64E_T
-# define LZO_TYPEOF_LZO_INT64L_T LZO_TYPEOF_LZO_INT64E_T
-#else
-#endif
-#endif
-#if defined(lzo_int64l_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64l_t) >= 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64l_t) == LZO_SIZEOF_LZO_INT64L_T)
-#endif
-#if !defined(lzo_int32f_t)
-#if (LZO_SIZEOF_SIZE_T >= 8)
-# define lzo_int32f_t lzo_int64l_t
-# define lzo_uint32f_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INT32F_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INT32F_T LZO_TYPEOF_LZO_INT64L_T
-#else
-# define lzo_int32f_t lzo_int32l_t
-# define lzo_uint32f_t lzo_uint32l_t
-# define LZO_SIZEOF_LZO_INT32F_T LZO_SIZEOF_LZO_INT32L_T
-# define LZO_TYPEOF_LZO_INT32F_T LZO_TYPEOF_LZO_INT32L_T
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32f_t) >= 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32f_t) == LZO_SIZEOF_LZO_INT32F_T)
-#endif
-#if !defined(lzo_int64f_t)
-#if defined(lzo_int64l_t)
-# define lzo_int64f_t lzo_int64l_t
-# define lzo_uint64f_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INT64F_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INT64F_T LZO_TYPEOF_LZO_INT64L_T
-#else
-#endif
-#endif
-#if defined(lzo_int64f_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64f_t) >= 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64f_t) == LZO_SIZEOF_LZO_INT64F_T)
-#endif
-#if !defined(lzo_intptr_t)
-#if 1 && (LZO_OS_OS400 && (LZO_SIZEOF_VOID_P == 16))
-# define __LZO_INTPTR_T_IS_POINTER 1
- typedef char * lzo_intptr_t;
- typedef char * lzo_uintptr_t;
-# define lzo_intptr_t lzo_intptr_t
-# define lzo_uintptr_t lzo_uintptr_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_VOID_P
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_CHAR_P
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300) && (LZO_SIZEOF_VOID_P == 4) && (LZO_SIZEOF_INT == 4))
- typedef __w64 int lzo_intptr_t;
- typedef __w64 unsigned int lzo_uintptr_t;
-# define lzo_intptr_t lzo_intptr_t
-# define lzo_uintptr_t lzo_uintptr_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == LZO_SIZEOF_VOID_P) && (LZO_SIZEOF_INT > LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t short
-# define lzo_uintptr_t unsigned short
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_SHORT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_SHORT
-#elif (LZO_SIZEOF_INT >= LZO_SIZEOF_VOID_P) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_intptr_t int
-# define lzo_uintptr_t unsigned int
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_LONG >= LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t long
-# define lzo_uintptr_t unsigned long
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_LZO_INT64L_T >= LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t lzo_int64l_t
-# define lzo_uintptr_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_LZO_INT64L_T
-#else
-# error "lzo_intptr_t"
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_intptr_t) >= sizeof(void *))
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_intptr_t) == sizeof(lzo_uintptr_t))
-#endif
-#if !defined(lzo_word_t)
-#if defined(LZO_WORDSIZE) && (LZO_WORDSIZE+0 > 0)
-#if (LZO_WORDSIZE == LZO_SIZEOF_LZO_INTPTR_T) && !(__LZO_INTPTR_T_IS_POINTER)
-# define lzo_word_t lzo_uintptr_t
-# define lzo_sword_t lzo_intptr_t
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LZO_INTPTR_T
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_LZO_INTPTR_T
-#elif (LZO_WORDSIZE == LZO_SIZEOF_LONG)
-# define lzo_word_t unsigned long
-# define lzo_sword_t long
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_LONG
-#elif (LZO_WORDSIZE == LZO_SIZEOF_INT)
-# define lzo_word_t unsigned int
-# define lzo_sword_t int
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_INT
-#elif (LZO_WORDSIZE == LZO_SIZEOF_SHORT)
-# define lzo_word_t unsigned short
-# define lzo_sword_t short
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_SHORT
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_SHORT
-#elif (LZO_WORDSIZE == 1)
-# define lzo_word_t unsigned char
-# define lzo_sword_t signed char
-# define LZO_SIZEOF_LZO_WORD_T 1
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_CHAR
-#elif (LZO_WORDSIZE == LZO_SIZEOF_LZO_INT64L_T)
-# define lzo_word_t lzo_uint64l_t
-# define lzo_sword_t lzo_int64l_t
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_WORD_T LZO_SIZEOF_LZO_INT64L_T
-#elif (LZO_ARCH_SPU) && (LZO_CC_GNUC)
-#if 0
- typedef unsigned lzo_word_t __attribute__((__mode__(__V16QI__)));
- typedef int lzo_sword_t __attribute__((__mode__(__V16QI__)));
-# define lzo_word_t lzo_word_t
-# define lzo_sword_t lzo_sword_t
-# define LZO_SIZEOF_LZO_WORD_T 16
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF___MODE_V16QI
-#endif
-#else
-# error "lzo_word_t"
-#endif
-#endif
-#endif
-#if 1 && defined(lzo_word_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_word_t) == LZO_WORDSIZE)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_sword_t) == LZO_WORDSIZE)
-#endif
-#if 1
-#define lzo_int8_t signed char
-#define lzo_uint8_t unsigned char
-#define LZO_SIZEOF_LZO_INT8_T 1
-#define LZO_TYPEOF_LZO_INT8_T LZO_TYPEOF_CHAR
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int8_t) == 1)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int8_t) == sizeof(lzo_uint8_t))
-#endif
-#if defined(lzo_int16e_t)
-#define lzo_int16_t lzo_int16e_t
-#define lzo_uint16_t lzo_uint16e_t
-#define LZO_SIZEOF_LZO_INT16_T LZO_SIZEOF_LZO_INT16E_T
-#define LZO_TYPEOF_LZO_INT16_T LZO_TYPEOF_LZO_INT16E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16_t) == 2)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16_t) == sizeof(lzo_uint16_t))
-#endif
-#if defined(lzo_int32e_t)
-#define lzo_int32_t lzo_int32e_t
-#define lzo_uint32_t lzo_uint32e_t
-#define LZO_SIZEOF_LZO_INT32_T LZO_SIZEOF_LZO_INT32E_T
-#define LZO_TYPEOF_LZO_INT32_T LZO_TYPEOF_LZO_INT32E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32_t) == 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32_t) == sizeof(lzo_uint32_t))
-#endif
-#if defined(lzo_int64e_t)
-#define lzo_int64_t lzo_int64e_t
-#define lzo_uint64_t lzo_uint64e_t
-#define LZO_SIZEOF_LZO_INT64_T LZO_SIZEOF_LZO_INT64E_T
-#define LZO_TYPEOF_LZO_INT64_T LZO_TYPEOF_LZO_INT64E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64_t) == 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64_t) == sizeof(lzo_uint64_t))
-#endif
-#if 1
-#define lzo_int_least32_t lzo_int32l_t
-#define lzo_uint_least32_t lzo_uint32l_t
-#define LZO_SIZEOF_LZO_INT_LEAST32_T LZO_SIZEOF_LZO_INT32L_T
-#define LZO_TYPEOF_LZO_INT_LEAST32_T LZO_TYPEOF_LZO_INT32L_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least32_t) >= 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least32_t) == sizeof(lzo_uint_least32_t))
-#endif
-#if defined(lzo_int64l_t)
-#define lzo_int_least64_t lzo_int64l_t
-#define lzo_uint_least64_t lzo_uint64l_t
-#define LZO_SIZEOF_LZO_INT_LEAST64_T LZO_SIZEOF_LZO_INT64L_T
-#define LZO_TYPEOF_LZO_INT_LEAST64_T LZO_TYPEOF_LZO_INT64L_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least64_t) >= 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least64_t) == sizeof(lzo_uint_least64_t))
-#endif
-#if 1
-#define lzo_int_fast32_t lzo_int32f_t
-#define lzo_uint_fast32_t lzo_uint32f_t
-#define LZO_SIZEOF_LZO_INT_FAST32_T LZO_SIZEOF_LZO_INT32F_T
-#define LZO_TYPEOF_LZO_INT_FAST32_T LZO_TYPEOF_LZO_INT32F_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast32_t) >= 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast32_t) == sizeof(lzo_uint_fast32_t))
-#endif
-#if defined(lzo_int64f_t)
-#define lzo_int_fast64_t lzo_int64f_t
-#define lzo_uint_fast64_t lzo_uint64f_t
-#define LZO_SIZEOF_LZO_INT_FAST64_T LZO_SIZEOF_LZO_INT64F_T
-#define LZO_TYPEOF_LZO_INT_FAST64_T LZO_TYPEOF_LZO_INT64F_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast64_t) >= 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast64_t) == sizeof(lzo_uint_fast64_t))
-#endif
-#if !defined(LZO_INT16_C)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 2)
-# define LZO_INT16_C(c) ((c) + 0)
-# define LZO_UINT16_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 2)
-# define LZO_INT16_C(c) ((c) + 0L)
-# define LZO_UINT16_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 2)
-# define LZO_INT16_C(c) (c)
-# define LZO_UINT16_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 2)
-# define LZO_INT16_C(c) (c##L)
-# define LZO_UINT16_C(c) (c##UL)
-# else
-# error "LZO_INT16_C"
-# endif
-#endif
-#if !defined(LZO_INT32_C)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 4)
-# define LZO_INT32_C(c) ((c) + 0)
-# define LZO_UINT32_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 4)
-# define LZO_INT32_C(c) ((c) + 0L)
-# define LZO_UINT32_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 4)
-# define LZO_INT32_C(c) (c)
-# define LZO_UINT32_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 4)
-# define LZO_INT32_C(c) (c##L)
-# define LZO_UINT32_C(c) (c##UL)
-# elif (LZO_SIZEOF_LONG_LONG >= 4)
-# define LZO_INT32_C(c) (c##LL)
-# define LZO_UINT32_C(c) (c##ULL)
-# else
-# error "LZO_INT32_C"
-# endif
-#endif
-#if !defined(LZO_INT64_C) && defined(lzo_int64l_t)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 8)
-# define LZO_INT64_C(c) ((c) + 0)
-# define LZO_UINT64_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 8)
-# define LZO_INT64_C(c) ((c) + 0L)
-# define LZO_UINT64_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 8)
-# define LZO_INT64_C(c) (c)
-# define LZO_UINT64_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 8)
-# define LZO_INT64_C(c) (c##L)
-# define LZO_UINT64_C(c) (c##UL)
-# else
-# error "LZO_INT64_C"
-# endif
-#endif
-#endif
-
-#endif /* already included */
-
-/* vim:set ts=4 sw=4 et: */
diff --git a/contrib/libs/minilzo/minilzo.c b/contrib/libs/minilzo/minilzo.c
deleted file mode 100644
index d05c0edbb7..0000000000
--- a/contrib/libs/minilzo/minilzo.c
+++ /dev/null
@@ -1,6234 +0,0 @@
-/* minilzo.c -- mini subset of the LZO real-time data compression library
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
- Markus F.X.J. Oberhumer
- <markus@oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-/*
- * NOTE:
- * the full LZO package can be found at
- * http://www.oberhumer.com/opensource/lzo/
- */
-
-/* This is Arcadia-specific header used for symbol mangling. */
-#include "protect.h"
-
-#define __LZO_IN_MINILZO 1
-
-#if defined(LZO_CFG_FREESTANDING)
-# undef MINILZO_HAVE_CONFIG_H
-# define LZO_LIBC_FREESTANDING 1
-# define LZO_OS_FREESTANDING 1
-#endif
-
-#ifdef MINILZO_HAVE_CONFIG_H
-#error # include <config.h>
-#endif
-#include <limits.h>
-#include <stddef.h>
-#if defined(MINILZO_CFG_USE_INTERNAL_LZODEFS)
-
-#ifndef __LZODEFS_H_INCLUDED
-#define __LZODEFS_H_INCLUDED 1
-
-#if defined(__CYGWIN32__) && !defined(__CYGWIN__)
-# define __CYGWIN__ __CYGWIN32__
-#endif
-#if 1 && defined(__INTERIX) && defined(__GNUC__) && !defined(_ALL_SOURCE)
-# define _ALL_SOURCE 1
-#endif
-#if defined(__mips__) && defined(__R5900__)
-# if !defined(__LONG_MAX__)
-# define __LONG_MAX__ 9223372036854775807L
-# endif
-#endif
-#if !defined(LZO_CFG_NO_DISABLE_WUNDEF)
-#if defined(__ARMCC_VERSION)
-# pragma diag_suppress 193
-#elif defined(__clang__) && defined(__clang_minor__)
-# pragma clang diagnostic ignored "-Wundef"
-#elif defined(__INTEL_COMPILER)
-# pragma warning(disable: 193)
-#elif defined(__KEIL__) && defined(__C166__)
-# pragma warning disable = 322
-#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && !defined(__PATHSCALE__)
-# if ((__GNUC__-0) >= 5 || ((__GNUC__-0) == 4 && (__GNUC_MINOR__-0) >= 2))
-# pragma GCC diagnostic ignored "-Wundef"
-# endif
-#elif defined(_MSC_VER) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__MWERKS__)
-# if ((_MSC_VER-0) >= 1300)
-# pragma warning(disable: 4668)
-# endif
-#endif
-#endif
-#if 0 && defined(__POCC__) && defined(_WIN32)
-# if (__POCC__ >= 400)
-# pragma warn(disable: 2216)
-# endif
-#endif
-#if 0 && defined(__WATCOMC__)
-# if (__WATCOMC__ >= 1050) && (__WATCOMC__ < 1060)
-# pragma warning 203 9
-# endif
-#endif
-#if defined(__BORLANDC__) && defined(__MSDOS__) && !defined(__FLAT__)
-# pragma option -h
-#endif
-#if !(LZO_CFG_NO_DISABLE_WCRTNONSTDC)
-#ifndef _CRT_NONSTDC_NO_DEPRECATE
-#define _CRT_NONSTDC_NO_DEPRECATE 1
-#endif
-#ifndef _CRT_NONSTDC_NO_WARNINGS
-#define _CRT_NONSTDC_NO_WARNINGS 1
-#endif
-#ifndef _CRT_SECURE_NO_DEPRECATE
-#define _CRT_SECURE_NO_DEPRECATE 1
-#endif
-#ifndef _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_WARNINGS 1
-#endif
-#endif
-#if 0
-#define LZO_0xffffUL 0xfffful
-#define LZO_0xffffffffUL 0xfffffffful
-#else
-#define LZO_0xffffUL 65535ul
-#define LZO_0xffffffffUL 4294967295ul
-#endif
-#define LZO_0xffffL LZO_0xffffUL
-#define LZO_0xffffffffL LZO_0xffffffffUL
-#if (LZO_0xffffL == LZO_0xffffffffL)
-# error "your preprocessor is broken 1"
-#endif
-#if (16ul * 16384ul != 262144ul)
-# error "your preprocessor is broken 2"
-#endif
-#if 0
-#if (32767 >= 4294967295ul)
-# error "your preprocessor is broken 3"
-#endif
-#if (65535u >= 4294967295ul)
-# error "your preprocessor is broken 4"
-#endif
-#endif
-#if defined(__COUNTER__)
-# ifndef LZO_CFG_USE_COUNTER
-# define LZO_CFG_USE_COUNTER 1
-# endif
-#else
-# undef LZO_CFG_USE_COUNTER
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-#if defined(__ZTC__) && defined(__I86__) && !defined(__OS2__)
-# if !defined(MSDOS)
-# define MSDOS 1
-# endif
-# if !defined(_MSDOS)
-# define _MSDOS 1
-# endif
-#elif 0 && defined(__VERSION) && defined(MB_LEN_MAX)
-# if (__VERSION == 520) && (MB_LEN_MAX == 1)
-# if !defined(__AZTEC_C__)
-# define __AZTEC_C__ __VERSION
-# endif
-# if !defined(__DOS__)
-# define __DOS__ 1
-# endif
-# endif
-#endif
-#endif
-#if defined(_MSC_VER) && defined(M_I86HM) && (UINT_MAX == LZO_0xffffL)
-# define ptrdiff_t long
-# define _PTRDIFF_T_DEFINED 1
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-# undef __LZO_RENAME_A
-# undef __LZO_RENAME_B
-# if defined(__AZTEC_C__) && defined(__DOS__)
-# define __LZO_RENAME_A 1
-# elif defined(_MSC_VER) && defined(MSDOS)
-# if (_MSC_VER < 600)
-# define __LZO_RENAME_A 1
-# elif (_MSC_VER < 700)
-# define __LZO_RENAME_B 1
-# endif
-# elif defined(__TSC__) && defined(__OS2__)
-# define __LZO_RENAME_A 1
-# elif defined(__MSDOS__) && defined(__TURBOC__) && (__TURBOC__ < 0x0410)
-# define __LZO_RENAME_A 1
-# elif defined(__PACIFIC__) && defined(DOS)
-# if !defined(__far)
-# define __far far
-# endif
-# if !defined(__near)
-# define __near near
-# endif
-# endif
-# if defined(__LZO_RENAME_A)
-# if !defined(__cdecl)
-# define __cdecl cdecl
-# endif
-# if !defined(__far)
-# define __far far
-# endif
-# if !defined(__huge)
-# define __huge huge
-# endif
-# if !defined(__near)
-# define __near near
-# endif
-# if !defined(__pascal)
-# define __pascal pascal
-# endif
-# if !defined(__huge)
-# define __huge huge
-# endif
-# elif defined(__LZO_RENAME_B)
-# if !defined(__cdecl)
-# define __cdecl _cdecl
-# endif
-# if !defined(__far)
-# define __far _far
-# endif
-# if !defined(__huge)
-# define __huge _huge
-# endif
-# if !defined(__near)
-# define __near _near
-# endif
-# if !defined(__pascal)
-# define __pascal _pascal
-# endif
-# elif (defined(__PUREC__) || defined(__TURBOC__)) && defined(__TOS__)
-# if !defined(__cdecl)
-# define __cdecl cdecl
-# endif
-# if !defined(__pascal)
-# define __pascal pascal
-# endif
-# endif
-# undef __LZO_RENAME_A
-# undef __LZO_RENAME_B
-#endif
-#if (UINT_MAX == LZO_0xffffL)
-#if defined(__AZTEC_C__) && defined(__DOS__)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-#elif defined(_MSC_VER) && defined(MSDOS)
-# if (_MSC_VER < 600)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-# endif
-# if (_MSC_VER < 700)
-# define LZO_BROKEN_INTEGRAL_PROMOTION 1
-# define LZO_BROKEN_SIZEOF 1
-# endif
-#elif defined(__PACIFIC__) && defined(DOS)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-#elif defined(__TURBOC__) && defined(__MSDOS__)
-# if (__TURBOC__ < 0x0150)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-# define LZO_BROKEN_INTEGRAL_PROMOTION 1
-# endif
-# if (__TURBOC__ < 0x0200)
-# define LZO_BROKEN_SIZEOF 1
-# endif
-# if (__TURBOC__ < 0x0400) && defined(__cplusplus)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# endif
-#elif (defined(__PUREC__) || defined(__TURBOC__)) && defined(__TOS__)
-# define LZO_BROKEN_CDECL_ALT_SYNTAX 1
-# define LZO_BROKEN_SIZEOF 1
-#endif
-#endif
-#if defined(__WATCOMC__) && (__WATCOMC__ < 900)
-# define LZO_BROKEN_INTEGRAL_CONSTANTS 1
-#endif
-#if defined(_CRAY) && defined(_CRAY1)
-# define LZO_BROKEN_SIGNED_RIGHT_SHIFT 1
-#endif
-#define LZO_PP_STRINGIZE(x) #x
-#define LZO_PP_MACRO_EXPAND(x) LZO_PP_STRINGIZE(x)
-#define LZO_PP_CONCAT0() /*empty*/
-#define LZO_PP_CONCAT1(a) a
-#define LZO_PP_CONCAT2(a,b) a ## b
-#define LZO_PP_CONCAT3(a,b,c) a ## b ## c
-#define LZO_PP_CONCAT4(a,b,c,d) a ## b ## c ## d
-#define LZO_PP_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e
-#define LZO_PP_CONCAT6(a,b,c,d,e,f) a ## b ## c ## d ## e ## f
-#define LZO_PP_CONCAT7(a,b,c,d,e,f,g) a ## b ## c ## d ## e ## f ## g
-#define LZO_PP_ECONCAT0() LZO_PP_CONCAT0()
-#define LZO_PP_ECONCAT1(a) LZO_PP_CONCAT1(a)
-#define LZO_PP_ECONCAT2(a,b) LZO_PP_CONCAT2(a,b)
-#define LZO_PP_ECONCAT3(a,b,c) LZO_PP_CONCAT3(a,b,c)
-#define LZO_PP_ECONCAT4(a,b,c,d) LZO_PP_CONCAT4(a,b,c,d)
-#define LZO_PP_ECONCAT5(a,b,c,d,e) LZO_PP_CONCAT5(a,b,c,d,e)
-#define LZO_PP_ECONCAT6(a,b,c,d,e,f) LZO_PP_CONCAT6(a,b,c,d,e,f)
-#define LZO_PP_ECONCAT7(a,b,c,d,e,f,g) LZO_PP_CONCAT7(a,b,c,d,e,f,g)
-#define LZO_PP_EMPTY /*empty*/
-#define LZO_PP_EMPTY0() /*empty*/
-#define LZO_PP_EMPTY1(a) /*empty*/
-#define LZO_PP_EMPTY2(a,b) /*empty*/
-#define LZO_PP_EMPTY3(a,b,c) /*empty*/
-#define LZO_PP_EMPTY4(a,b,c,d) /*empty*/
-#define LZO_PP_EMPTY5(a,b,c,d,e) /*empty*/
-#define LZO_PP_EMPTY6(a,b,c,d,e,f) /*empty*/
-#define LZO_PP_EMPTY7(a,b,c,d,e,f,g) /*empty*/
-#if 1
-#define LZO_CPP_STRINGIZE(x) #x
-#define LZO_CPP_MACRO_EXPAND(x) LZO_CPP_STRINGIZE(x)
-#define LZO_CPP_CONCAT2(a,b) a ## b
-#define LZO_CPP_CONCAT3(a,b,c) a ## b ## c
-#define LZO_CPP_CONCAT4(a,b,c,d) a ## b ## c ## d
-#define LZO_CPP_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e
-#define LZO_CPP_CONCAT6(a,b,c,d,e,f) a ## b ## c ## d ## e ## f
-#define LZO_CPP_CONCAT7(a,b,c,d,e,f,g) a ## b ## c ## d ## e ## f ## g
-#define LZO_CPP_ECONCAT2(a,b) LZO_CPP_CONCAT2(a,b)
-#define LZO_CPP_ECONCAT3(a,b,c) LZO_CPP_CONCAT3(a,b,c)
-#define LZO_CPP_ECONCAT4(a,b,c,d) LZO_CPP_CONCAT4(a,b,c,d)
-#define LZO_CPP_ECONCAT5(a,b,c,d,e) LZO_CPP_CONCAT5(a,b,c,d,e)
-#define LZO_CPP_ECONCAT6(a,b,c,d,e,f) LZO_CPP_CONCAT6(a,b,c,d,e,f)
-#define LZO_CPP_ECONCAT7(a,b,c,d,e,f,g) LZO_CPP_CONCAT7(a,b,c,d,e,f,g)
-#endif
-#define __LZO_MASK_GEN(o,b) (((((o) << ((b)-!!(b))) - (o)) << 1) + (o)*!!(b))
-#if 1 && defined(__cplusplus)
-# if !defined(__STDC_CONSTANT_MACROS)
-# define __STDC_CONSTANT_MACROS 1
-# endif
-# if !defined(__STDC_LIMIT_MACROS)
-# define __STDC_LIMIT_MACROS 1
-# endif
-#endif
-#if defined(__cplusplus)
-# define LZO_EXTERN_C extern "C"
-# define LZO_EXTERN_C_BEGIN extern "C" {
-# define LZO_EXTERN_C_END }
-#else
-# define LZO_EXTERN_C extern
-# define LZO_EXTERN_C_BEGIN /*empty*/
-# define LZO_EXTERN_C_END /*empty*/
-#endif
-#if !defined(__LZO_OS_OVERRIDE)
-#if (LZO_OS_FREESTANDING)
-# define LZO_INFO_OS "freestanding"
-#elif (LZO_OS_EMBEDDED)
-# define LZO_INFO_OS "embedded"
-#elif 1 && defined(__IAR_SYSTEMS_ICC__)
-# define LZO_OS_EMBEDDED 1
-# define LZO_INFO_OS "embedded"
-#elif defined(__CYGWIN__) && defined(__GNUC__)
-# define LZO_OS_CYGWIN 1
-# define LZO_INFO_OS "cygwin"
-#elif defined(__EMX__) && defined(__GNUC__)
-# define LZO_OS_EMX 1
-# define LZO_INFO_OS "emx"
-#elif defined(__BEOS__)
-# define LZO_OS_BEOS 1
-# define LZO_INFO_OS "beos"
-#elif defined(__Lynx__)
-# define LZO_OS_LYNXOS 1
-# define LZO_INFO_OS "lynxos"
-#elif defined(__OS400__)
-# define LZO_OS_OS400 1
-# define LZO_INFO_OS "os400"
-#elif defined(__QNX__)
-# define LZO_OS_QNX 1
-# define LZO_INFO_OS "qnx"
-#elif defined(__BORLANDC__) && defined(__DPMI32__) && (__BORLANDC__ >= 0x0460)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-#elif defined(__BORLANDC__) && defined(__DPMI16__)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-#elif defined(__ZTC__) && defined(DOS386)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-#elif defined(__OS2__) || defined(__OS2V2__)
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_OS216 1
-# define LZO_INFO_OS "os216"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_OS2 1
-# define LZO_INFO_OS "os2"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__WIN64__) || defined(_WIN64) || defined(WIN64)
-# define LZO_OS_WIN64 1
-# define LZO_INFO_OS "win64"
-#elif defined(__WIN32__) || defined(_WIN32) || defined(WIN32) || defined(__WINDOWS_386__)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-#elif defined(__MWERKS__) && defined(__INTEL__)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-#elif defined(__WINDOWS__) || defined(_WINDOWS) || defined(_Windows)
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_WIN16 1
-# define LZO_INFO_OS "win16"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__DOS__) || defined(__MSDOS__) || defined(_MSDOS) || defined(MSDOS) || (defined(__PACIFIC__) && defined(DOS))
-# if (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_OS_DOS32 1
-# define LZO_INFO_OS "dos32"
-# else
-# error "check your limits.h header"
-# endif
-#elif defined(__WATCOMC__)
-# if defined(__NT__) && (UINT_MAX == LZO_0xffffL)
-# define LZO_OS_DOS16 1
-# define LZO_INFO_OS "dos16"
-# elif defined(__NT__) && (__WATCOMC__ < 1100)
-# define LZO_OS_WIN32 1
-# define LZO_INFO_OS "win32"
-# elif defined(__linux__) || defined(__LINUX__)
-# define LZO_OS_POSIX 1
-# define LZO_INFO_OS "posix"
-# else
-# error "please specify a target using the -bt compiler option"
-# endif
-#elif defined(__palmos__)
-# define LZO_OS_PALMOS 1
-# define LZO_INFO_OS "palmos"
-#elif defined(__TOS__) || defined(__atarist__)
-# define LZO_OS_TOS 1
-# define LZO_INFO_OS "tos"
-#elif defined(macintosh) && !defined(__arm__) && !defined(__i386__) && !defined(__ppc__) && !defined(__x64_64__)
-# define LZO_OS_MACCLASSIC 1
-# define LZO_INFO_OS "macclassic"
-#elif defined(__VMS)
-# define LZO_OS_VMS 1
-# define LZO_INFO_OS "vms"
-#elif (defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__)
-# define LZO_OS_CONSOLE 1
-# define LZO_OS_CONSOLE_PS2 1
-# define LZO_INFO_OS "console"
-# define LZO_INFO_OS_CONSOLE "ps2"
-#elif defined(__mips__) && defined(__psp__)
-# define LZO_OS_CONSOLE 1
-# define LZO_OS_CONSOLE_PSP 1
-# define LZO_INFO_OS "console"
-# define LZO_INFO_OS_CONSOLE "psp"
-#else
-# define LZO_OS_POSIX 1
-# define LZO_INFO_OS "posix"
-#endif
-#if (LZO_OS_POSIX)
-# if defined(_AIX) || defined(__AIX__) || defined(__aix__)
-# define LZO_OS_POSIX_AIX 1
-# define LZO_INFO_OS_POSIX "aix"
-# elif defined(__FreeBSD__)
-# define LZO_OS_POSIX_FREEBSD 1
-# define LZO_INFO_OS_POSIX "freebsd"
-# elif defined(__hpux__) || defined(__hpux)
-# define LZO_OS_POSIX_HPUX 1
-# define LZO_INFO_OS_POSIX "hpux"
-# elif defined(__INTERIX)
-# define LZO_OS_POSIX_INTERIX 1
-# define LZO_INFO_OS_POSIX "interix"
-# elif defined(__IRIX__) || defined(__irix__)
-# define LZO_OS_POSIX_IRIX 1
-# define LZO_INFO_OS_POSIX "irix"
-# elif defined(__linux__) || defined(__linux) || defined(__LINUX__)
-# define LZO_OS_POSIX_LINUX 1
-# define LZO_INFO_OS_POSIX "linux"
-# elif defined(__APPLE__) && defined(__MACH__)
-# if ((__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__-0) >= 20000)
-# define LZO_OS_POSIX_DARWIN 1040
-# define LZO_INFO_OS_POSIX "darwin_iphone"
-# elif ((__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__-0) >= 1040)
-# define LZO_OS_POSIX_DARWIN __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__
-# define LZO_INFO_OS_POSIX "darwin"
-# else
-# define LZO_OS_POSIX_DARWIN 1
-# define LZO_INFO_OS_POSIX "darwin"
-# endif
-# define LZO_OS_POSIX_MACOSX LZO_OS_POSIX_DARWIN
-# elif defined(__minix__) || defined(__minix)
-# define LZO_OS_POSIX_MINIX 1
-# define LZO_INFO_OS_POSIX "minix"
-# elif defined(__NetBSD__)
-# define LZO_OS_POSIX_NETBSD 1
-# define LZO_INFO_OS_POSIX "netbsd"
-# elif defined(__OpenBSD__)
-# define LZO_OS_POSIX_OPENBSD 1
-# define LZO_INFO_OS_POSIX "openbsd"
-# elif defined(__osf__)
-# define LZO_OS_POSIX_OSF 1
-# define LZO_INFO_OS_POSIX "osf"
-# elif defined(__solaris__) || defined(__sun)
-# if defined(__SVR4) || defined(__svr4__)
-# define LZO_OS_POSIX_SOLARIS 1
-# define LZO_INFO_OS_POSIX "solaris"
-# else
-# define LZO_OS_POSIX_SUNOS 1
-# define LZO_INFO_OS_POSIX "sunos"
-# endif
-# elif defined(__ultrix__) || defined(__ultrix)
-# define LZO_OS_POSIX_ULTRIX 1
-# define LZO_INFO_OS_POSIX "ultrix"
-# elif defined(_UNICOS)
-# define LZO_OS_POSIX_UNICOS 1
-# define LZO_INFO_OS_POSIX "unicos"
-# else
-# define LZO_OS_POSIX_UNKNOWN 1
-# define LZO_INFO_OS_POSIX "unknown"
-# endif
-#endif
-#endif
-#if (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-# if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_OS_DOS32 || LZO_OS_OS2 || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (UINT_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if defined(CIL) && defined(_GNUCC) && defined(__GNUC__)
-# define LZO_CC_CILLY 1
-# define LZO_INFO_CC "Cilly"
-# if defined(__CILLY__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__CILLY__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif 0 && defined(SDCC) && defined(__VERSION__) && !defined(__GNUC__)
-# define LZO_CC_SDCC 1
-# define LZO_INFO_CC "sdcc"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(SDCC)
-#elif defined(__PATHSCALE__) && defined(__PATHCC_PATCHLEVEL__)
-# define LZO_CC_PATHSCALE (__PATHCC__ * 0x10000L + (__PATHCC_MINOR__-0) * 0x100 + (__PATHCC_PATCHLEVEL__-0))
-# define LZO_INFO_CC "Pathscale C"
-# define LZO_INFO_CCVER __PATHSCALE__
-# if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_PATHSCALE_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__INTEL_COMPILER) && ((__INTEL_COMPILER-0) > 0)
-# define LZO_CC_INTELC __INTEL_COMPILER
-# define LZO_INFO_CC "Intel C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__INTEL_COMPILER)
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_INTELC_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_INTELC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__POCC__) && defined(_WIN32)
-# define LZO_CC_PELLESC 1
-# define LZO_INFO_CC "Pelles C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__POCC__)
-#elif defined(__ARMCC_VERSION) && defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# if defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_ARMCC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# else
-# define LZO_CC_ARMCC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# endif
-# define LZO_CC_ARMCC __ARMCC_VERSION
-# define LZO_INFO_CC "ARM C Compiler"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__clang__) && defined(__llvm__) && defined(__VERSION__)
-# if defined(__clang_major__) && defined(__clang_minor__) && defined(__clang_patchlevel__)
-# define LZO_CC_CLANG (__clang_major__ * 0x10000L + (__clang_minor__-0) * 0x100 + (__clang_patchlevel__-0))
-# else
-# define LZO_CC_CLANG 0x010000L
-# endif
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_CLANG_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_CLANG_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-# define LZO_INFO_CC "clang"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__llvm__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# if defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_LLVM_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# else
-# define LZO_CC_LLVM_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# endif
-# define LZO_CC_LLVM LZO_CC_LLVM_GNUC
-# define LZO_INFO_CC "llvm-gcc"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(__ACK__) && defined(_ACK)
-# define LZO_CC_ACK 1
-# define LZO_INFO_CC "Amsterdam Compiler Kit C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__ARMCC_VERSION) && !defined(__GNUC__)
-# define LZO_CC_ARMCC __ARMCC_VERSION
-# define LZO_CC_ARMCC_ARMCC __ARMCC_VERSION
-# define LZO_INFO_CC "ARM C Compiler"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__ARMCC_VERSION)
-#elif defined(__AZTEC_C__)
-# define LZO_CC_AZTECC 1
-# define LZO_INFO_CC "Aztec C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__AZTEC_C__)
-#elif defined(__CODEGEARC__)
-# define LZO_CC_CODEGEARC 1
-# define LZO_INFO_CC "CodeGear C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__CODEGEARC__)
-#elif defined(__BORLANDC__)
-# define LZO_CC_BORLANDC 1
-# define LZO_INFO_CC "Borland C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__BORLANDC__)
-#elif defined(_CRAYC) && defined(_RELEASE)
-# define LZO_CC_CRAYC 1
-# define LZO_INFO_CC "Cray C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_RELEASE)
-#elif defined(__DMC__) && defined(__SC__)
-# define LZO_CC_DMC 1
-# define LZO_INFO_CC "Digital Mars C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__DMC__)
-#elif defined(__DECC)
-# define LZO_CC_DECC 1
-# define LZO_INFO_CC "DEC C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__DECC)
-#elif (defined(__ghs) || defined(__ghs__)) && defined(__GHS_VERSION_NUMBER) && ((__GHS_VERSION_NUMBER-0) > 0)
-# define LZO_CC_GHS 1
-# define LZO_INFO_CC "Green Hills C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__GHS_VERSION_NUMBER)
-# if defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_GHS_MSC _MSC_VER
-# elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__VERSION__)
-# define LZO_CC_GHS_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# endif
-#elif defined(__HIGHC__)
-# define LZO_CC_HIGHC 1
-# define LZO_INFO_CC "MetaWare High C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__HP_aCC) && ((__HP_aCC-0) > 0)
-# define LZO_CC_HPACC __HP_aCC
-# define LZO_INFO_CC "HP aCC"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__HP_aCC)
-#elif defined(__IAR_SYSTEMS_ICC__)
-# define LZO_CC_IARC 1
-# define LZO_INFO_CC "IAR C"
-# if defined(__VER__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__VER__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__IBMC__) && ((__IBMC__-0) > 0)
-# define LZO_CC_IBMC __IBMC__
-# define LZO_INFO_CC "IBM C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__IBMC__)
-#elif defined(__IBMCPP__) && ((__IBMCPP__-0) > 0)
-# define LZO_CC_IBMC __IBMCPP__
-# define LZO_INFO_CC "IBM C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__IBMCPP__)
-#elif defined(__KEIL__) && defined(__C166__)
-# define LZO_CC_KEILC 1
-# define LZO_INFO_CC "Keil C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__C166__)
-#elif defined(__LCC__) && defined(_WIN32) && defined(__LCCOPTIMLEVEL)
-# define LZO_CC_LCCWIN32 1
-# define LZO_INFO_CC "lcc-win32"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__LCC__)
-# define LZO_CC_LCC 1
-# define LZO_INFO_CC "lcc"
-# if defined(__LCC_VERSION__)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__LCC_VERSION__)
-# else
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__MWERKS__) && ((__MWERKS__-0) > 0)
-# define LZO_CC_MWERKS __MWERKS__
-# define LZO_INFO_CC "Metrowerks C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__MWERKS__)
-#elif (defined(__NDPC__) || defined(__NDPX__)) && defined(__i386)
-# define LZO_CC_NDPC 1
-# define LZO_INFO_CC "Microway NDP C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__PACIFIC__)
-# define LZO_CC_PACIFICC 1
-# define LZO_INFO_CC "Pacific C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PACIFIC__)
-#elif defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__)
-# if defined(__PGIC_PATCHLEVEL__)
-# define LZO_CC_PGI (__PGIC__ * 0x10000L + (__PGIC_MINOR__-0) * 0x100 + (__PGIC_PATCHLEVEL__-0))
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PGIC__) "." LZO_PP_MACRO_EXPAND(__PGIC_MINOR__) "." LZO_PP_MACRO_EXPAND(__PGIC_PATCHLEVEL__)
-# else
-# define LZO_CC_PGI (__PGIC__ * 0x10000L + (__PGIC_MINOR__-0) * 0x100)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PGIC__) "." LZO_PP_MACRO_EXPAND(__PGIC_MINOR__) ".0"
-# endif
-# define LZO_INFO_CC "Portland Group PGI C"
-#elif defined(__PGI) && (defined(__linux__) || defined(__WIN32__))
-# define LZO_CC_PGI 1
-# define LZO_INFO_CC "Portland Group PGI C"
-# define LZO_INFO_CCVER "unknown"
-#elif defined(__PUREC__) && defined(__TOS__)
-# define LZO_CC_PUREC 1
-# define LZO_INFO_CC "Pure C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__PUREC__)
-#elif defined(__SC__) && defined(__ZTC__)
-# define LZO_CC_SYMANTECC 1
-# define LZO_INFO_CC "Symantec C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SC__)
-#elif defined(__SUNPRO_C)
-# define LZO_INFO_CC "SunPro C"
-# if ((__SUNPRO_C-0) > 0)
-# define LZO_CC_SUNPROC __SUNPRO_C
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SUNPRO_C)
-# else
-# define LZO_CC_SUNPROC 1
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__SUNPRO_CC)
-# define LZO_INFO_CC "SunPro C"
-# if ((__SUNPRO_CC-0) > 0)
-# define LZO_CC_SUNPROC __SUNPRO_CC
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__SUNPRO_CC)
-# else
-# define LZO_CC_SUNPROC 1
-# define LZO_INFO_CCVER "unknown"
-# endif
-#elif defined(__TINYC__)
-# define LZO_CC_TINYC 1
-# define LZO_INFO_CC "Tiny C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TINYC__)
-#elif defined(__TSC__)
-# define LZO_CC_TOPSPEEDC 1
-# define LZO_INFO_CC "TopSpeed C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TSC__)
-#elif defined(__WATCOMC__)
-# define LZO_CC_WATCOMC 1
-# define LZO_INFO_CC "Watcom C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__WATCOMC__)
-#elif defined(__TURBOC__)
-# define LZO_CC_TURBOC 1
-# define LZO_INFO_CC "Turbo C"
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__TURBOC__)
-#elif defined(__ZTC__)
-# define LZO_CC_ZORTECHC 1
-# define LZO_INFO_CC "Zortech C"
-# if ((__ZTC__-0) == 0x310)
-# define LZO_INFO_CCVER "0x310"
-# else
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(__ZTC__)
-# endif
-#elif defined(__GNUC__) && defined(__VERSION__)
-# if defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100 + (__GNUC_PATCHLEVEL__-0))
-# elif defined(__GNUC_MINOR__)
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L + (__GNUC_MINOR__-0) * 0x100)
-# else
-# define LZO_CC_GNUC (__GNUC__ * 0x10000L)
-# endif
-# define LZO_INFO_CC "gcc"
-# define LZO_INFO_CCVER __VERSION__
-#elif defined(_MSC_VER) && ((_MSC_VER-0) > 0)
-# define LZO_CC_MSC _MSC_VER
-# define LZO_INFO_CC "Microsoft C"
-# if defined(_MSC_FULL_VER)
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_MSC_VER) "." LZO_PP_MACRO_EXPAND(_MSC_FULL_VER)
-# else
-# define LZO_INFO_CCVER LZO_PP_MACRO_EXPAND(_MSC_VER)
-# endif
-#else
-# define LZO_CC_UNKNOWN 1
-# define LZO_INFO_CC "unknown"
-# define LZO_INFO_CCVER "unknown"
-#endif
-#if (LZO_CC_GNUC) && defined(__OPEN64__)
-# if defined(__OPENCC__) && defined(__OPENCC_MINOR__) && defined(__OPENCC_PATCHLEVEL__)
-# define LZO_CC_OPEN64 (__OPENCC__ * 0x10000L + (__OPENCC_MINOR__-0) * 0x100 + (__OPENCC_PATCHLEVEL__-0))
-# define LZO_CC_OPEN64_GNUC LZO_CC_GNUC
-# endif
-#endif
-#if (LZO_CC_GNUC) && defined(__PCC__)
-# if defined(__PCC__) && defined(__PCC_MINOR__) && defined(__PCC_MINORMINOR__)
-# define LZO_CC_PCC (__PCC__ * 0x10000L + (__PCC_MINOR__-0) * 0x100 + (__PCC_MINORMINOR__-0))
-# define LZO_CC_PCC_GNUC LZO_CC_GNUC
-# endif
-#endif
-#if 0 && (LZO_CC_MSC && (_MSC_VER >= 1200)) && !defined(_MSC_FULL_VER)
-# error "LZO_CC_MSC: _MSC_FULL_VER is not defined"
-#endif
-#if !defined(__LZO_ARCH_OVERRIDE) && !(LZO_ARCH_GENERIC) && defined(_CRAY)
-# if (UINT_MAX > LZO_0xffffffffL) && defined(_CRAY)
-# if defined(_CRAYMPP) || defined(_CRAYT3D) || defined(_CRAYT3E)
-# define LZO_ARCH_CRAY_MPP 1
-# elif defined(_CRAY1)
-# define LZO_ARCH_CRAY_PVP 1
-# endif
-# endif
-#endif
-#if !defined(__LZO_ARCH_OVERRIDE)
-#if (LZO_ARCH_GENERIC)
-# define LZO_INFO_ARCH "generic"
-#elif (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-# define LZO_ARCH_I086 1
-# define LZO_INFO_ARCH "i086"
-#elif defined(__aarch64__)
-# define LZO_ARCH_ARM64 1
-# define LZO_INFO_ARCH "arm64"
-#elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA)
-# define LZO_ARCH_ALPHA 1
-# define LZO_INFO_ARCH "alpha"
-#elif (LZO_ARCH_CRAY_MPP) && (defined(_CRAYT3D) || defined(_CRAYT3E))
-# define LZO_ARCH_ALPHA 1
-# define LZO_INFO_ARCH "alpha"
-#elif defined(__amd64__) || defined(__x86_64__) || defined(_M_AMD64)
-# define LZO_ARCH_AMD64 1
-# define LZO_INFO_ARCH "amd64"
-#elif defined(__arm__) || defined(_M_ARM)
-# define LZO_ARCH_ARM 1
-# define LZO_INFO_ARCH "arm"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCARM__)
-# define LZO_ARCH_ARM 1
-# define LZO_INFO_ARCH "arm"
-#elif (UINT_MAX <= LZO_0xffffL) && defined(__AVR__)
-# define LZO_ARCH_AVR 1
-# define LZO_INFO_ARCH "avr"
-#elif defined(__avr32__) || defined(__AVR32__)
-# define LZO_ARCH_AVR32 1
-# define LZO_INFO_ARCH "avr32"
-#elif defined(__bfin__)
-# define LZO_ARCH_BLACKFIN 1
-# define LZO_INFO_ARCH "blackfin"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C166__)
-# define LZO_ARCH_C166 1
-# define LZO_INFO_ARCH "c166"
-#elif defined(__cris__)
-# define LZO_ARCH_CRIS 1
-# define LZO_INFO_ARCH "cris"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCEZ80__)
-# define LZO_ARCH_EZ80 1
-# define LZO_INFO_ARCH "ez80"
-#elif defined(__H8300__) || defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_ARCH_H8300 1
-# define LZO_INFO_ARCH "h8300"
-#elif defined(__hppa__) || defined(__hppa)
-# define LZO_ARCH_HPPA 1
-# define LZO_INFO_ARCH "hppa"
-#elif defined(__386__) || defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_M_I386)
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif (LZO_CC_ZORTECHC && defined(__I86__))
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif (LZO_OS_DOS32 && LZO_CC_HIGHC) && defined(_I386)
-# define LZO_ARCH_I386 1
-# define LZO_ARCH_IA32 1
-# define LZO_INFO_ARCH "i386"
-#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
-# define LZO_ARCH_IA64 1
-# define LZO_INFO_ARCH "ia64"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__m32c__)
-# define LZO_ARCH_M16C 1
-# define LZO_INFO_ARCH "m16c"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICCM16C__)
-# define LZO_ARCH_M16C 1
-# define LZO_INFO_ARCH "m16c"
-#elif defined(__m32r__)
-# define LZO_ARCH_M32R 1
-# define LZO_INFO_ARCH "m32r"
-#elif (LZO_OS_TOS) || defined(__m68k__) || defined(__m68000__) || defined(__mc68000__) || defined(__mc68020__) || defined(_M_M68K)
-# define LZO_ARCH_M68K 1
-# define LZO_INFO_ARCH "m68k"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C251__)
-# define LZO_ARCH_MCS251 1
-# define LZO_INFO_ARCH "mcs251"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__C51__)
-# define LZO_ARCH_MCS51 1
-# define LZO_INFO_ARCH "mcs51"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICC8051__)
-# define LZO_ARCH_MCS51 1
-# define LZO_INFO_ARCH "mcs51"
-#elif defined(__mips__) || defined(__mips) || defined(_MIPS_ARCH) || defined(_M_MRX000)
-# define LZO_ARCH_MIPS 1
-# define LZO_INFO_ARCH "mips"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__MSP430__)
-# define LZO_ARCH_MSP430 1
-# define LZO_INFO_ARCH "msp430"
-#elif defined(__IAR_SYSTEMS_ICC__) && defined(__ICC430__)
-# define LZO_ARCH_MSP430 1
-# define LZO_INFO_ARCH "msp430"
-#elif defined(__powerpc__) || defined(__powerpc) || defined(__ppc__) || defined(__PPC__) || defined(_M_PPC) || defined(_ARCH_PPC) || defined(_ARCH_PWR)
-# define LZO_ARCH_POWERPC 1
-# define LZO_INFO_ARCH "powerpc"
-#elif defined(__s390__) || defined(__s390) || defined(__s390x__) || defined(__s390x)
-# define LZO_ARCH_S390 1
-# define LZO_INFO_ARCH "s390"
-#elif defined(__sh__) || defined(_M_SH)
-# define LZO_ARCH_SH 1
-# define LZO_INFO_ARCH "sh"
-#elif defined(__sparc__) || defined(__sparc) || defined(__sparcv8)
-# define LZO_ARCH_SPARC 1
-# define LZO_INFO_ARCH "sparc"
-#elif defined(__SPU__)
-# define LZO_ARCH_SPU 1
-# define LZO_INFO_ARCH "spu"
-#elif (UINT_MAX == LZO_0xffffL) && defined(__z80)
-# define LZO_ARCH_Z80 1
-# define LZO_INFO_ARCH "z80"
-#elif (LZO_ARCH_CRAY_PVP)
-# if defined(_CRAYSV1)
-# define LZO_ARCH_CRAY_SV1 1
-# define LZO_INFO_ARCH "cray_sv1"
-# elif (_ADDR64)
-# define LZO_ARCH_CRAY_T90 1
-# define LZO_INFO_ARCH "cray_t90"
-# elif (_ADDR32)
-# define LZO_ARCH_CRAY_YMP 1
-# define LZO_INFO_ARCH "cray_ymp"
-# else
-# define LZO_ARCH_CRAY_XMP 1
-# define LZO_INFO_ARCH "cray_xmp"
-# endif
-#else
-# define LZO_ARCH_UNKNOWN 1
-# define LZO_INFO_ARCH "unknown"
-#endif
-#endif
-#if !defined(LZO_ARCH_ARM_THUMB2)
-#if (LZO_ARCH_ARM)
-# if defined(__ARM_ARCH_ISA_THUMB)
-# if ((__ARM_ARCH_ISA_THUMB)+0 >= 2)
-# define LZO_ARCH_ARM_THUMB2 1
-# endif
-# elif 1 && defined(__thumb2__)
-# define LZO_ARCH_ARM_THUMB2 1
-# elif 1 && defined(__TARGET_ARCH_THUMB) && ((__TARGET_ARCH_THUMB)+0 >= 4)
-# define LZO_ARCH_ARM_THUMB2 1
-# endif
-#endif
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_DOS32 || LZO_OS_OS2)
-# error "FIXME - missing define for CPU architecture"
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_WIN32)
-# error "FIXME - missing LZO_OS_WIN32 define for CPU architecture"
-#endif
-#if 1 && (LZO_ARCH_UNKNOWN) && (LZO_OS_WIN64)
-# error "FIXME - missing LZO_OS_WIN64 define for CPU architecture"
-#endif
-#if (LZO_OS_OS216 || LZO_OS_WIN16)
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && defined(BLX286))
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && defined(DOSX286))
-# define LZO_ARCH_I086PM 1
-#elif 1 && (LZO_OS_DOS16 && LZO_CC_BORLANDC && defined(__DPMI16__))
-# define LZO_ARCH_I086PM 1
-#endif
-#if (LZO_ARCH_AMD64 && !LZO_ARCH_X64)
-# define LZO_ARCH_X64 1
-#elif (!LZO_ARCH_AMD64 && LZO_ARCH_X64) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_AMD64 1
-#endif
-#if (LZO_ARCH_ARM64 && !LZO_ARCH_AARCH64)
-# define LZO_ARCH_AARCH64 1
-#elif (!LZO_ARCH_ARM64 && LZO_ARCH_AARCH64) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_ARM64 1
-#endif
-#if (LZO_ARCH_I386 && !LZO_ARCH_X86)
-# define LZO_ARCH_X86 1
-#elif (!LZO_ARCH_I386 && LZO_ARCH_X86) && defined(__LZO_ARCH_OVERRIDE)
-# define LZO_ARCH_I386 1
-#endif
-#if (LZO_ARCH_AMD64 && !LZO_ARCH_X64) || (!LZO_ARCH_AMD64 && LZO_ARCH_X64)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM64 && !LZO_ARCH_AARCH64) || (!LZO_ARCH_ARM64 && LZO_ARCH_AARCH64)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I386 && !LZO_ARCH_X86) || (!LZO_ARCH_I386 && LZO_ARCH_X86)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB1 && !LZO_ARCH_ARM)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB2 && !LZO_ARCH_ARM)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM_THUMB1 && LZO_ARCH_ARM_THUMB2)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I086PM && !LZO_ARCH_I086)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_I086)
-# if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_ARCH_I386)
-# if (UINT_MAX != LZO_0xffffL) && defined(__i386_int16__)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (UINT_MAX != LZO_0xffffffffL) && !defined(__i386_int16__)
-# error "unexpected configuration - check your compiler defines"
-# endif
-# if (ULONG_MAX != LZO_0xffffffffL)
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#if (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-# if !defined(LZO_TARGET_FEATURE_SSE2)
-# if defined(__SSE2__)
-# define LZO_TARGET_FEATURE_SSE2 1
-# elif defined(_MSC_VER) && (defined(_M_IX86_FP) && ((_M_IX86_FP)+0 >= 2))
-# define LZO_TARGET_FEATURE_SSE2 1
-# elif (LZO_CC_INTELC_MSC || LZO_CC_MSC) && defined(_M_AMD64)
-# define LZO_TARGET_FEATURE_SSE2 1
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_SSSE3)
-# if (LZO_TARGET_FEATURE_SSE2)
-# if defined(__SSSE3__)
-# define LZO_TARGET_FEATURE_SSSE3 1
-# elif defined(_MSC_VER) && defined(__AVX__)
-# define LZO_TARGET_FEATURE_SSSE3 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_SSE4_2)
-# if (LZO_TARGET_FEATURE_SSSE3)
-# if defined(__SSE4_2__)
-# define LZO_TARGET_FEATURE_SSE4_2 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_AVX)
-# if (LZO_TARGET_FEATURE_SSSE3)
-# if defined(__AVX__)
-# define LZO_TARGET_FEATURE_AVX 1
-# endif
-# endif
-# endif
-# if !defined(LZO_TARGET_FEATURE_AVX2)
-# if (LZO_TARGET_FEATURE_AVX)
-# if defined(__AVX2__)
-# define LZO_TARGET_FEATURE_AVX2 1
-# endif
-# endif
-# endif
-#endif
-#if (LZO_TARGET_FEATURE_SSSE3 && !(LZO_TARGET_FEATURE_SSE2))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_SSE4_2 && !(LZO_TARGET_FEATURE_SSSE3))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_AVX && !(LZO_TARGET_FEATURE_SSSE3))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_TARGET_FEATURE_AVX2 && !(LZO_TARGET_FEATURE_AVX))
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ARCH_ARM)
-# if !defined(LZO_TARGET_FEATURE_NEON)
-# if defined(__ARM_NEON) && ((__ARM_NEON)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# elif 1 && defined(__ARM_NEON__) && ((__ARM_NEON__)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# elif 1 && defined(__TARGET_FEATURE_NEON) && ((__TARGET_FEATURE_NEON)+0)
-# define LZO_TARGET_FEATURE_NEON 1
-# endif
-# endif
-#elif (LZO_ARCH_ARM64)
-# if !defined(LZO_TARGET_FEATURE_NEON)
-# if 1
-# define LZO_TARGET_FEATURE_NEON 1
-# endif
-# endif
-#endif
-#if 0
-#elif !defined(__LZO_MM_OVERRIDE)
-#if (LZO_ARCH_I086)
-#if (UINT_MAX != LZO_0xffffL)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if defined(__TINY__) || defined(M_I86TM) || defined(_M_I86TM)
-# define LZO_MM_TINY 1
-#elif defined(__HUGE__) || defined(_HUGE_) || defined(M_I86HM) || defined(_M_I86HM)
-# define LZO_MM_HUGE 1
-#elif defined(__SMALL__) || defined(M_I86SM) || defined(_M_I86SM) || defined(SMALL_MODEL)
-# define LZO_MM_SMALL 1
-#elif defined(__MEDIUM__) || defined(M_I86MM) || defined(_M_I86MM)
-# define LZO_MM_MEDIUM 1
-#elif defined(__COMPACT__) || defined(M_I86CM) || defined(_M_I86CM)
-# define LZO_MM_COMPACT 1
-#elif defined(__LARGE__) || defined(M_I86LM) || defined(_M_I86LM) || defined(LARGE_MODEL)
-# define LZO_MM_LARGE 1
-#elif (LZO_CC_AZTECC)
-# if defined(_LARGE_CODE) && defined(_LARGE_DATA)
-# define LZO_MM_LARGE 1
-# elif defined(_LARGE_CODE)
-# define LZO_MM_MEDIUM 1
-# elif defined(_LARGE_DATA)
-# define LZO_MM_COMPACT 1
-# else
-# define LZO_MM_SMALL 1
-# endif
-#elif (LZO_CC_ZORTECHC && defined(__VCM__))
-# define LZO_MM_LARGE 1
-#else
-# error "unknown LZO_ARCH_I086 memory model"
-#endif
-#if (LZO_OS_DOS16 || LZO_OS_OS216 || LZO_OS_WIN16)
-#define LZO_HAVE_MM_HUGE_PTR 1
-#define LZO_HAVE_MM_HUGE_ARRAY 1
-#if (LZO_MM_TINY)
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#endif
-#if (LZO_CC_AZTECC || LZO_CC_PACIFICC || LZO_CC_ZORTECHC)
-# undef LZO_HAVE_MM_HUGE_PTR
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#elif (LZO_CC_DMC || LZO_CC_SYMANTECC)
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#elif (LZO_CC_MSC && defined(_QC))
-# undef LZO_HAVE_MM_HUGE_ARRAY
-# if (_MSC_VER < 600)
-# undef LZO_HAVE_MM_HUGE_PTR
-# endif
-#elif (LZO_CC_TURBOC && (__TURBOC__ < 0x0295))
-# undef LZO_HAVE_MM_HUGE_ARRAY
-#endif
-#if (LZO_ARCH_I086PM) && !(LZO_HAVE_MM_HUGE_PTR)
-# if (LZO_OS_DOS16)
-# error "unexpected configuration - check your compiler defines"
-# elif (LZO_CC_ZORTECHC)
-# else
-# error "unexpected configuration - check your compiler defines"
-# endif
-#endif
-#ifdef __cplusplus
-extern "C" {
-#endif
-#if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0200))
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_DMC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_MSC || LZO_CC_TOPSPEEDC)
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif (LZO_CC_TURBOC && (__TURBOC__ >= 0x0295))
- extern void __near __cdecl _AHSHIFT(void);
-# define LZO_MM_AHSHIFT ((unsigned) _AHSHIFT)
-#elif ((LZO_CC_AZTECC || LZO_CC_PACIFICC || LZO_CC_TURBOC) && LZO_OS_DOS16)
-# define LZO_MM_AHSHIFT 12
-#elif (LZO_CC_WATCOMC)
- extern unsigned char _HShift;
-# define LZO_MM_AHSHIFT ((unsigned) _HShift)
-#else
-# error "FIXME - implement LZO_MM_AHSHIFT"
-#endif
-#ifdef __cplusplus
-}
-#endif
-#endif
-#elif (LZO_ARCH_C166)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_C166 __MODEL__"
-#elif ((__MODEL__) == 0)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 1)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_C166 __MODEL__"
-#endif
-#elif (LZO_ARCH_MCS251)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_MCS251 __MODEL__"
-#elif ((__MODEL__) == 0)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_MCS251 __MODEL__"
-#endif
-#elif (LZO_ARCH_MCS51)
-#if !defined(__MODEL__)
-# error "FIXME - LZO_ARCH_MCS51 __MODEL__"
-#elif ((__MODEL__) == 1)
-# define LZO_MM_SMALL 1
-#elif ((__MODEL__) == 2)
-# define LZO_MM_LARGE 1
-#elif ((__MODEL__) == 3)
-# define LZO_MM_TINY 1
-#elif ((__MODEL__) == 4)
-# define LZO_MM_XTINY 1
-#elif ((__MODEL__) == 5)
-# define LZO_MM_XSMALL 1
-#else
-# error "FIXME - LZO_ARCH_MCS51 __MODEL__"
-#endif
-#elif (LZO_ARCH_CRAY_PVP)
-# define LZO_MM_PVP 1
-#else
-# define LZO_MM_FLAT 1
-#endif
-#if (LZO_MM_COMPACT)
-# define LZO_INFO_MM "compact"
-#elif (LZO_MM_FLAT)
-# define LZO_INFO_MM "flat"
-#elif (LZO_MM_HUGE)
-# define LZO_INFO_MM "huge"
-#elif (LZO_MM_LARGE)
-# define LZO_INFO_MM "large"
-#elif (LZO_MM_MEDIUM)
-# define LZO_INFO_MM "medium"
-#elif (LZO_MM_PVP)
-# define LZO_INFO_MM "pvp"
-#elif (LZO_MM_SMALL)
-# define LZO_INFO_MM "small"
-#elif (LZO_MM_TINY)
-# define LZO_INFO_MM "tiny"
-#else
-# error "unknown memory model"
-#endif
-#endif
-#if !defined(__lzo_gnuc_extension__)
-#if (LZO_CC_GNUC >= 0x020800ul)
-# define __lzo_gnuc_extension__ __extension__
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_gnuc_extension__ __extension__
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_gnuc_extension__ __extension__
-#else
-#endif
-#endif
-#if !defined(__lzo_gnuc_extension__)
-# define __lzo_gnuc_extension__ /*empty*/
-#endif
-#if !defined(lzo_has_builtin)
-#if (LZO_CC_CLANG) && defined(__has_builtin)
-# define lzo_has_builtin __has_builtin
-#endif
-#endif
-#if !defined(lzo_has_builtin)
-# define lzo_has_builtin(x) 0
-#endif
-#if !defined(lzo_has_attribute)
-#if (LZO_CC_CLANG) && defined(__has_attribute)
-# define lzo_has_attribute __has_attribute
-#endif
-#endif
-#if !defined(lzo_has_attribute)
-# define lzo_has_attribute(x) 0
-#endif
-#if !defined(lzo_has_declspec_attribute)
-#if (LZO_CC_CLANG) && defined(__has_declspec_attribute)
-# define lzo_has_declspec_attribute __has_declspec_attribute
-#endif
-#endif
-#if !defined(lzo_has_declspec_attribute)
-# define lzo_has_declspec_attribute(x) 0
-#endif
-#if !defined(lzo_has_feature)
-#if (LZO_CC_CLANG) && defined(__has_feature)
-# define lzo_has_feature __has_feature
-#endif
-#endif
-#if !defined(lzo_has_feature)
-# define lzo_has_feature(x) 0
-#endif
-#if !defined(lzo_has_extension)
-#if (LZO_CC_CLANG) && defined(__has_extension)
-# define lzo_has_extension __has_extension
-#elif (LZO_CC_CLANG) && defined(__has_feature)
-# define lzo_has_extension __has_feature
-#endif
-#endif
-#if !defined(lzo_has_extension)
-# define lzo_has_extension 0
-#endif
-#if !defined(LZO_CFG_USE_NEW_STYLE_CASTS) && defined(__cplusplus) && 0
-# if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul))
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-# elif (LZO_CC_INTELC && (__INTEL_COMPILER < 1200))
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-# else
-# define LZO_CFG_USE_NEW_STYLE_CASTS 1
-# endif
-#endif
-#if !defined(LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-#endif
-#if !defined(__cplusplus)
-# if defined(LZO_CFG_USE_NEW_STYLE_CASTS)
-# undef LZO_CFG_USE_NEW_STYLE_CASTS
-# endif
-# define LZO_CFG_USE_NEW_STYLE_CASTS 0
-#endif
-#if !defined(LZO_REINTERPRET_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_REINTERPRET_CAST(t,e) (reinterpret_cast<t> (e))
-# endif
-#endif
-#if !defined(LZO_REINTERPRET_CAST)
-# define LZO_REINTERPRET_CAST(t,e) ((t) (e))
-#endif
-#if !defined(LZO_STATIC_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_STATIC_CAST(t,e) (static_cast<t> (e))
-# endif
-#endif
-#if !defined(LZO_STATIC_CAST)
-# define LZO_STATIC_CAST(t,e) ((t) (e))
-#endif
-#if !defined(LZO_STATIC_CAST2)
-# define LZO_STATIC_CAST2(t1,t2,e) LZO_STATIC_CAST(t1, LZO_STATIC_CAST(t2, e))
-#endif
-#if !defined(LZO_UNCONST_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNCONST_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNCONST_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNCONST_CAST(t,e) ((t) ((void *) ((lzo_uintptr_t) ((const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNCONST_CAST)
-# define LZO_UNCONST_CAST(t,e) ((t) ((void *) ((const void *) (e))))
-#endif
-#if !defined(LZO_UNCONST_VOLATILE_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) ((volatile void *) ((lzo_uintptr_t) ((volatile const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNCONST_VOLATILE_CAST)
-# define LZO_UNCONST_VOLATILE_CAST(t,e) ((t) ((volatile void *) ((volatile const void *) (e))))
-#endif
-#if !defined(LZO_UNVOLATILE_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNVOLATILE_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) ((void *) ((lzo_uintptr_t) ((volatile void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNVOLATILE_CAST)
-# define LZO_UNVOLATILE_CAST(t,e) ((t) ((void *) ((volatile void *) (e))))
-#endif
-#if !defined(LZO_UNVOLATILE_CONST_CAST)
-# if (LZO_CFG_USE_NEW_STYLE_CASTS)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) (const_cast<t> (e))
-# elif (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) (e))
-# elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) ((const void *) ((lzo_uintptr_t) ((volatile const void *) (e)))))
-# endif
-#endif
-#if !defined(LZO_UNVOLATILE_CONST_CAST)
-# define LZO_UNVOLATILE_CONST_CAST(t,e) ((t) ((const void *) ((volatile const void *) (e))))
-#endif
-#if !defined(LZO_PCAST)
-# if (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_PCAST(t,e) ((t) (e))
-# endif
-#endif
-#if !defined(LZO_PCAST)
-# define LZO_PCAST(t,e) LZO_STATIC_CAST(t, LZO_STATIC_CAST(void *, e))
-#endif
-#if !defined(LZO_CCAST)
-# if (LZO_HAVE_MM_HUGE_PTR)
-# define LZO_CCAST(t,e) ((t) (e))
-# endif
-#endif
-#if !defined(LZO_CCAST)
-# define LZO_CCAST(t,e) LZO_STATIC_CAST(t, LZO_STATIC_CAST(const void *, e))
-#endif
-#if !defined(LZO_ICONV)
-# define LZO_ICONV(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(LZO_ICAST)
-# define LZO_ICAST(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(LZO_ITRUNC)
-# define LZO_ITRUNC(t,e) LZO_STATIC_CAST(t, e)
-#endif
-#if !defined(__lzo_cte)
-# if (LZO_CC_MSC || LZO_CC_WATCOMC)
-# define __lzo_cte(e) ((void)0,(e))
-# elif 1
-# define __lzo_cte(e) ((void)0,(e))
-# endif
-#endif
-#if !defined(__lzo_cte)
-# define __lzo_cte(e) (e)
-#endif
-#if !defined(LZO_BLOCK_BEGIN)
-# define LZO_BLOCK_BEGIN do {
-# define LZO_BLOCK_END } while __lzo_cte(0)
-#endif
-#if !defined(LZO_UNUSED)
-# if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0600))
-# define LZO_UNUSED(var) ((void) &var)
-# elif (LZO_CC_BORLANDC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PELLESC || LZO_CC_TURBOC)
-# define LZO_UNUSED(var) if (&var) ; else
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x030200ul))
-# define LZO_UNUSED(var) ((void) &var)
-# elif (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_UNUSED(var) ((void) var)
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_UNUSED(var) if (&var) ; else
-# elif (LZO_CC_KEILC)
-# define LZO_UNUSED(var) {extern int lzo_unused__[1-2*!(sizeof(var)>0)]; (void)lzo_unused__;}
-# elif (LZO_CC_PACIFICC)
-# define LZO_UNUSED(var) ((void) sizeof(var))
-# elif (LZO_CC_WATCOMC) && defined(__cplusplus)
-# define LZO_UNUSED(var) ((void) var)
-# else
-# define LZO_UNUSED(var) ((void) &var)
-# endif
-#endif
-#if !defined(LZO_UNUSED_FUNC)
-# if (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0600))
-# define LZO_UNUSED_FUNC(func) ((void) func)
-# elif (LZO_CC_BORLANDC || LZO_CC_NDPC || LZO_CC_TURBOC)
-# define LZO_UNUSED_FUNC(func) if (func) ; else
-# elif (LZO_CC_CLANG || LZO_CC_LLVM)
-# define LZO_UNUSED_FUNC(func) ((void) &func)
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_UNUSED_FUNC(func) if (func) ; else
-# elif (LZO_CC_MSC)
-# define LZO_UNUSED_FUNC(func) ((void) &func)
-# elif (LZO_CC_KEILC || LZO_CC_PELLESC)
-# define LZO_UNUSED_FUNC(func) {extern int lzo_unused__[1-2*!(sizeof((int)func)>0)]; (void)lzo_unused__;}
-# else
-# define LZO_UNUSED_FUNC(func) ((void) func)
-# endif
-#endif
-#if !defined(LZO_UNUSED_LABEL)
-# if (LZO_CC_CLANG >= 0x020800ul)
-# define LZO_UNUSED_LABEL(l) (__lzo_gnuc_extension__ ((void) ((const void *) &&l)))
-# elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_INTELC || LZO_CC_WATCOMC)
-# define LZO_UNUSED_LABEL(l) if __lzo_cte(0) goto l
-# else
-# define LZO_UNUSED_LABEL(l) switch (0) case 1:goto l
-# endif
-#endif
-#if !defined(LZO_DEFINE_UNINITIALIZED_VAR)
-# if 0
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var
-# elif 0 && (LZO_CC_GNUC)
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var = var
-# else
-# define LZO_DEFINE_UNINITIALIZED_VAR(type,var,init) type var = init
-# endif
-#endif
-#if !defined(__lzo_inline)
-#if (LZO_CC_TURBOC && (__TURBOC__ <= 0x0295))
-#elif defined(__cplusplus)
-# define __lzo_inline inline
-#elif defined(__STDC_VERSION__) && (__STDC_VERSION__-0 >= 199901L)
-# define __lzo_inline inline
-#elif (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0550))
-# define __lzo_inline __inline
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define __lzo_inline __inline__
-#elif (LZO_CC_DMC)
-# define __lzo_inline __inline
-#elif (LZO_CC_GHS)
-# define __lzo_inline __inline__
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_inline __inline__
-#elif (LZO_CC_INTELC)
-# define __lzo_inline __inline
-#elif (LZO_CC_MWERKS && (__MWERKS__ >= 0x2405))
-# define __lzo_inline __inline
-#elif (LZO_CC_MSC && (_MSC_VER >= 900))
-# define __lzo_inline __inline
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_inline __inline__
-#endif
-#endif
-#if defined(__lzo_inline)
-# ifndef __lzo_HAVE_inline
-# define __lzo_HAVE_inline 1
-# endif
-#else
-# define __lzo_inline /*empty*/
-#endif
-#if !defined(__lzo_forceinline)
-#if (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450))
-# define __lzo_forceinline __forceinline
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200))
-# define __lzo_forceinline __forceinline
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_forceinline __inline__ __attribute__((__always_inline__))
-#endif
-#endif
-#if defined(__lzo_forceinline)
-# ifndef __lzo_HAVE_forceinline
-# define __lzo_HAVE_forceinline 1
-# endif
-#else
-# define __lzo_forceinline __lzo_inline
-#endif
-#if !defined(__lzo_noinline)
-#if 1 && (LZO_ARCH_I386) && (LZO_CC_GNUC >= 0x040000ul) && (LZO_CC_GNUC < 0x040003ul)
-# define __lzo_noinline __attribute__((__noinline__,__used__))
-#elif (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 600))
-# define __lzo_noinline __declspec(noinline)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_noinline __declspec(noinline)
-#elif (LZO_CC_MWERKS && (__MWERKS__ >= 0x3200) && (LZO_OS_WIN32 || LZO_OS_WIN64))
-# if defined(__cplusplus)
-# else
-# define __lzo_noinline __declspec(noinline)
-# endif
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_noinline __attribute__((__noinline__))
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_noinline __attribute__((__noinline__))
-#endif
-#endif
-#if defined(__lzo_noinline)
-# ifndef __lzo_HAVE_noinline
-# define __lzo_HAVE_noinline 1
-# endif
-#else
-# define __lzo_noinline /*empty*/
-#endif
-#if (__lzo_HAVE_forceinline || __lzo_HAVE_noinline) && !(__lzo_HAVE_inline)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if !defined(__lzo_static_inline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_inline __lzo_gnuc_extension__ static __lzo_inline
-#endif
-#endif
-#if !defined(__lzo_static_inline)
-# define __lzo_static_inline static __lzo_inline
-#endif
-#if !defined(__lzo_static_forceinline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_forceinline __lzo_gnuc_extension__ static __lzo_forceinline
-#endif
-#endif
-#if !defined(__lzo_static_forceinline)
-# define __lzo_static_forceinline static __lzo_forceinline
-#endif
-#if !defined(__lzo_static_noinline)
-#if (LZO_CC_IBMC)
-# define __lzo_static_noinline __lzo_gnuc_extension__ static __lzo_noinline
-#endif
-#endif
-#if !defined(__lzo_static_noinline)
-# define __lzo_static_noinline static __lzo_noinline
-#endif
-#if !defined(__lzo_c99_extern_inline)
-#if defined(__GNUC_GNU_INLINE__)
-# define __lzo_c99_extern_inline __lzo_inline
-#elif defined(__GNUC_STDC_INLINE__)
-# define __lzo_c99_extern_inline extern __lzo_inline
-#elif defined(__STDC_VERSION__) && (__STDC_VERSION__-0 >= 199901L)
-# define __lzo_c99_extern_inline extern __lzo_inline
-#endif
-#if !defined(__lzo_c99_extern_inline) && (__lzo_HAVE_inline)
-# define __lzo_c99_extern_inline __lzo_inline
-#endif
-#endif
-#if defined(__lzo_c99_extern_inline)
-# ifndef __lzo_HAVE_c99_extern_inline
-# define __lzo_HAVE_c99_extern_inline 1
-# endif
-#else
-# define __lzo_c99_extern_inline /*empty*/
-#endif
-#if !defined(__lzo_may_alias)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_CLANG >= 0x020900ul)
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 1210)) && 0
-# define __lzo_may_alias __attribute__((__may_alias__))
-#elif (LZO_CC_PGI >= 0x0d0a00ul) && 0
-# define __lzo_may_alias __attribute__((__may_alias__))
-#endif
-#endif
-#if defined(__lzo_may_alias)
-# ifndef __lzo_HAVE_may_alias
-# define __lzo_HAVE_may_alias 1
-# endif
-#else
-# define __lzo_may_alias /*empty*/
-#endif
-#if !defined(__lzo_noreturn)
-#if (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450))
-# define __lzo_noreturn __declspec(noreturn)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 600))
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200))
-# define __lzo_noreturn __declspec(noreturn)
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_noreturn __attribute__((__noreturn__))
-#endif
-#endif
-#if defined(__lzo_noreturn)
-# ifndef __lzo_HAVE_noreturn
-# define __lzo_HAVE_noreturn 1
-# endif
-#else
-# define __lzo_noreturn /*empty*/
-#endif
-#if !defined(__lzo_nothrow)
-#if (LZO_CC_GNUC >= 0x030300ul)
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 450)) && defined(__cplusplus)
-# define __lzo_nothrow __declspec(nothrow)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 900))
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_nothrow __attribute__((__nothrow__))
-#elif (LZO_CC_MSC && (_MSC_VER >= 1200)) && defined(__cplusplus)
-# define __lzo_nothrow __declspec(nothrow)
-#endif
-#endif
-#if defined(__lzo_nothrow)
-# ifndef __lzo_HAVE_nothrow
-# define __lzo_HAVE_nothrow 1
-# endif
-#else
-# define __lzo_nothrow /*empty*/
-#endif
-#if !defined(__lzo_restrict)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_IBMC >= 800) && !defined(__cplusplus)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_IBMC >= 1210)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 600))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 600))
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM)
-# define __lzo_restrict __restrict__
-#elif (LZO_CC_MSC && (_MSC_VER >= 1400))
-# define __lzo_restrict __restrict
-#elif (LZO_CC_PGI >= 0x0d0a00ul)
-# define __lzo_restrict __restrict__
-#endif
-#endif
-#if defined(__lzo_restrict)
-# ifndef __lzo_HAVE_restrict
-# define __lzo_HAVE_restrict 1
-# endif
-#else
-# define __lzo_restrict /*empty*/
-#endif
-#if !defined(__lzo_alignof)
-#if (LZO_CC_ARMCC || LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_GHS) && !defined(__cplusplus)
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_IBMC >= 600)
-# define __lzo_alignof(e) (__lzo_gnuc_extension__ __alignof__(e))
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER >= 700))
-# define __lzo_alignof(e) __alignof__(e)
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_alignof(e) __alignof(e)
-#elif (LZO_CC_SUNPROC >= 0x5100)
-# define __lzo_alignof(e) __alignof__(e)
-#endif
-#endif
-#if defined(__lzo_alignof)
-# ifndef __lzo_HAVE_alignof
-# define __lzo_HAVE_alignof 1
-# endif
-#endif
-#if !defined(__lzo_struct_packed)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020700ul))
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_PCC && (LZO_CC_PCC < 0x010100ul))
-#elif (LZO_CC_SUNPROC && (LZO_CC_SUNPROC < 0x5110)) && !defined(__cplusplus)
-#elif (LZO_CC_GNUC >= 0x030400ul) && !(LZO_CC_PCC_GNUC) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-# define __lzo_struct_packed(s) struct s {
-# define __lzo_struct_packed_end() } __attribute__((__gcc_struct__,__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__gcc_struct__,__packed__));
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || (LZO_CC_PGI >= 0x0d0a00ul) || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_struct_packed(s) struct s {
-# define __lzo_struct_packed_end() } __attribute__((__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_struct_packed(s) __lzo_gnuc_extension__ struct s {
-# define __lzo_struct_packed_end() } __attribute__((__packed__));
-# define __lzo_struct_packed_ma_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_struct_packed(s) __pragma(pack(push,1)) struct s {
-# define __lzo_struct_packed_end() } __pragma(pack(pop));
-#elif (LZO_CC_WATCOMC && (__WATCOMC__ >= 900))
-# define __lzo_struct_packed(s) _Packed struct s {
-# define __lzo_struct_packed_end() };
-#endif
-#endif
-#if defined(__lzo_struct_packed) && !defined(__lzo_struct_packed_ma)
-# define __lzo_struct_packed_ma(s) __lzo_struct_packed(s)
-#endif
-#if defined(__lzo_struct_packed_end) && !defined(__lzo_struct_packed_ma_end)
-# define __lzo_struct_packed_ma_end() __lzo_struct_packed_end()
-#endif
-#if !defined(__lzo_byte_struct)
-#if defined(__lzo_struct_packed)
-# define __lzo_byte_struct(s,n) __lzo_struct_packed(s) unsigned char a[n]; __lzo_struct_packed_end()
-# define __lzo_byte_struct_ma(s,n) __lzo_struct_packed_ma(s) unsigned char a[n]; __lzo_struct_packed_ma_end()
-#elif (LZO_CC_CILLY || LZO_CC_CLANG || LZO_CC_PGI || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_byte_struct(s,n) struct s { unsigned char a[n]; } __attribute__((__packed__));
-# define __lzo_byte_struct_ma(s,n) struct s { unsigned char a[n]; } __lzo_may_alias __attribute__((__packed__));
-#endif
-#endif
-#if defined(__lzo_byte_struct) && !defined(__lzo_byte_struct_ma)
-# define __lzo_byte_struct_ma(s,n) __lzo_byte_struct(s,n)
-#endif
-#if !defined(__lzo_struct_align16) && (__lzo_HAVE_alignof)
-#if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x030000ul))
-#elif (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_CILLY || LZO_CC_PCC)
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_struct_align16(s) struct __declspec(align(16)) s {
-# define __lzo_struct_align16_end() };
-# define __lzo_struct_align32(s) struct __declspec(align(32)) s {
-# define __lzo_struct_align32_end() };
-# define __lzo_struct_align64(s) struct __declspec(align(64)) s {
-# define __lzo_struct_align64_end() };
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || (LZO_CC_IBMC >= 700) || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_struct_align16(s) struct s {
-# define __lzo_struct_align16_end() } __attribute__((__aligned__(16)));
-# define __lzo_struct_align32(s) struct s {
-# define __lzo_struct_align32_end() } __attribute__((__aligned__(32)));
-# define __lzo_struct_align64(s) struct s {
-# define __lzo_struct_align64_end() } __attribute__((__aligned__(64)));
-#endif
-#endif
-#if !defined(__lzo_union_um)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020700ul))
-#elif (LZO_CC_GNUC && (LZO_CC_GNUC < 0x020800ul)) && defined(__cplusplus)
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER < 810))
-#elif (LZO_CC_PCC && (LZO_CC_PCC < 0x010100ul))
-#elif (LZO_CC_SUNPROC && (LZO_CC_SUNPROC < 0x5110)) && !defined(__cplusplus)
-#elif (LZO_CC_ARMCC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || (LZO_CC_PGI >= 0x0d0a00ul) || (LZO_CC_SUNPROC >= 0x5100))
-# define __lzo_union_am(s) union s {
-# define __lzo_union_am_end() } __lzo_may_alias;
-# define __lzo_union_um(s) union s {
-# define __lzo_union_um_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_IBMC >= 700)
-# define __lzo_union_am(s) __lzo_gnuc_extension__ union s {
-# define __lzo_union_am_end() } __lzo_may_alias;
-# define __lzo_union_um(s) __lzo_gnuc_extension__ union s {
-# define __lzo_union_um_end() } __lzo_may_alias __attribute__((__packed__));
-#elif (LZO_CC_INTELC_MSC) || (LZO_CC_MSC && (_MSC_VER >= 1300))
-# define __lzo_union_um(s) __pragma(pack(push,1)) union s {
-# define __lzo_union_um_end() } __pragma(pack(pop));
-#elif (LZO_CC_WATCOMC && (__WATCOMC__ >= 900))
-# define __lzo_union_um(s) _Packed union s {
-# define __lzo_union_um_end() };
-#endif
-#endif
-#if !defined(__lzo_union_am)
-# define __lzo_union_am(s) union s {
-# define __lzo_union_am_end() };
-#endif
-#if !defined(__lzo_constructor)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_constructor __attribute__((__constructor__,__used__))
-#elif (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_constructor __attribute__((__constructor__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_constructor __attribute__((__constructor__,__used__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_constructor __attribute__((__constructor__))
-#endif
-#endif
-#if defined(__lzo_constructor)
-# ifndef __lzo_HAVE_constructor
-# define __lzo_HAVE_constructor 1
-# endif
-#endif
-#if !defined(__lzo_destructor)
-#if (LZO_CC_GNUC >= 0x030400ul)
-# define __lzo_destructor __attribute__((__destructor__,__used__))
-#elif (LZO_CC_GNUC >= 0x020700ul)
-# define __lzo_destructor __attribute__((__destructor__))
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 800))
-# define __lzo_destructor __attribute__((__destructor__,__used__))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_destructor __attribute__((__destructor__))
-#endif
-#endif
-#if defined(__lzo_destructor)
-# ifndef __lzo_HAVE_destructor
-# define __lzo_HAVE_destructor 1
-# endif
-#endif
-#if (__lzo_HAVE_destructor) && !(__lzo_HAVE_constructor)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if !defined(__lzo_likely) && !defined(__lzo_unlikely)
-#if (LZO_CC_GNUC >= 0x030200ul)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_IBMC >= 1010)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER >= 800))
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define __lzo_likely(e) (__builtin_expect(!!(e),1))
-# define __lzo_unlikely(e) (__builtin_expect(!!(e),0))
-#endif
-#endif
-#if defined(__lzo_likely)
-# ifndef __lzo_HAVE_likely
-# define __lzo_HAVE_likely 1
-# endif
-#else
-# define __lzo_likely(e) (e)
-#endif
-#if defined(__lzo_very_likely)
-# ifndef __lzo_HAVE_very_likely
-# define __lzo_HAVE_very_likely 1
-# endif
-#else
-# define __lzo_very_likely(e) __lzo_likely(e)
-#endif
-#if defined(__lzo_unlikely)
-# ifndef __lzo_HAVE_unlikely
-# define __lzo_HAVE_unlikely 1
-# endif
-#else
-# define __lzo_unlikely(e) (e)
-#endif
-#if defined(__lzo_very_unlikely)
-# ifndef __lzo_HAVE_very_unlikely
-# define __lzo_HAVE_very_unlikely 1
-# endif
-#else
-# define __lzo_very_unlikely(e) __lzo_unlikely(e)
-#endif
-#if !defined(__lzo_loop_forever)
-# if (LZO_CC_IBMC)
-# define __lzo_loop_forever() LZO_BLOCK_BEGIN for (;;) { ; } LZO_BLOCK_END
-# else
-# define __lzo_loop_forever() do { ; } while __lzo_cte(1)
-# endif
-#endif
-#if !defined(__lzo_unreachable)
-#if (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x020800ul)) && lzo_has_builtin(__builtin_unreachable)
-# define __lzo_unreachable() __builtin_unreachable();
-#elif (LZO_CC_GNUC >= 0x040500ul)
-# define __lzo_unreachable() __builtin_unreachable();
-#elif (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 1300)) && 1
-# define __lzo_unreachable() __builtin_unreachable();
-#endif
-#endif
-#if defined(__lzo_unreachable)
-# ifndef __lzo_HAVE_unreachable
-# define __lzo_HAVE_unreachable 1
-# endif
-#else
-# if 0
-# define __lzo_unreachable() ((void)0);
-# else
-# define __lzo_unreachable() __lzo_loop_forever();
-# endif
-#endif
-#if !defined(lzo_unused_funcs_impl)
-# if 1 && (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || (LZO_CC_GNUC >= 0x020700ul) || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-# define lzo_unused_funcs_impl(r,f) static r __attribute__((__unused__)) f
-# elif 1 && (LZO_CC_BORLANDC || LZO_CC_GNUC)
-# define lzo_unused_funcs_impl(r,f) static r f
-# else
-# define lzo_unused_funcs_impl(r,f) __lzo_static_forceinline r f
-# endif
-#endif
-#ifndef __LZO_CTA_NAME
-#if (LZO_CFG_USE_COUNTER)
-# define __LZO_CTA_NAME(a) LZO_PP_ECONCAT2(a,__COUNTER__)
-#else
-# define __LZO_CTA_NAME(a) LZO_PP_ECONCAT2(a,__LINE__)
-#endif
-#endif
-#if !defined(LZO_COMPILE_TIME_ASSERT_HEADER)
-# if (LZO_CC_AZTECC || LZO_CC_ZORTECHC)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_DMC || LZO_CC_SYMANTECC)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1u-2*!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_TURBOC && (__TURBOC__ == 0x0295))
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-!(e)]; LZO_EXTERN_C_END
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG < 0x020900ul)) && defined(__cplusplus)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN int __LZO_CTA_NAME(lzo_cta_f__)(int [1-2*!(e)]); LZO_EXTERN_C_END
-# elif (LZO_CC_GNUC) && defined(__CHECKER__) && defined(__SPARSE_CHECKER__)
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN enum {__LZO_CTA_NAME(lzo_cta_e__)=1/!!(e)} __attribute__((__unused__)); LZO_EXTERN_C_END
-# else
-# define LZO_COMPILE_TIME_ASSERT_HEADER(e) LZO_EXTERN_C_BEGIN extern int __LZO_CTA_NAME(lzo_cta__)[1-2*!(e)]; LZO_EXTERN_C_END
-# endif
-#endif
-#if !defined(LZO_COMPILE_TIME_ASSERT)
-# if (LZO_CC_AZTECC)
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-!(e)];}
-# elif (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x030000ul))
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)] __attribute__((__unused__));}
-# elif (LZO_CC_DMC || LZO_CC_PACIFICC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# elif (LZO_CC_GNUC) && defined(__CHECKER__) && defined(__SPARSE_CHECKER__)
-# define LZO_COMPILE_TIME_ASSERT(e) {(void) (0/!!(e));}
-# elif (LZO_CC_GNUC >= 0x040700ul) && (LZO_CFG_USE_COUNTER) && defined(__cplusplus)
-# define LZO_COMPILE_TIME_ASSERT(e) {enum {__LZO_CTA_NAME(lzo_cta_e__)=1/!!(e)} __attribute__((__unused__));}
-# elif (LZO_CC_GNUC >= 0x040700ul)
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)] __attribute__((__unused__));}
-# elif (LZO_CC_MSC && (_MSC_VER < 900))
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# elif (LZO_CC_TURBOC && (__TURBOC__ == 0x0295))
-# define LZO_COMPILE_TIME_ASSERT(e) switch(0) case 1:case !(e):break;
-# else
-# define LZO_COMPILE_TIME_ASSERT(e) {typedef int __LZO_CTA_NAME(lzo_cta_t__)[1-2*!(e)];}
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(1 == 1)
-#if defined(__cplusplus)
-extern "C" { LZO_COMPILE_TIME_ASSERT_HEADER(2 == 2) }
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(3 == 3)
-#if (LZO_ARCH_I086 || LZO_ARCH_I386) && (LZO_OS_DOS16 || LZO_OS_DOS32 || LZO_OS_OS2 || LZO_OS_OS216 || LZO_OS_WIN16 || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (LZO_CC_GNUC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PACIFICC)
-# elif (LZO_CC_DMC || LZO_CC_SYMANTECC || LZO_CC_ZORTECHC)
-# define __lzo_cdecl __cdecl
-# define __lzo_cdecl_atexit /*empty*/
-# define __lzo_cdecl_main __cdecl
-# if (LZO_OS_OS2 && (LZO_CC_DMC || LZO_CC_SYMANTECC))
-# define __lzo_cdecl_qsort __pascal
-# elif (LZO_OS_OS2 && (LZO_CC_ZORTECHC))
-# define __lzo_cdecl_qsort _stdcall
-# else
-# define __lzo_cdecl_qsort __cdecl
-# endif
-# elif (LZO_CC_WATCOMC)
-# define __lzo_cdecl __cdecl
-# else
-# define __lzo_cdecl __cdecl
-# define __lzo_cdecl_atexit __cdecl
-# define __lzo_cdecl_main __cdecl
-# define __lzo_cdecl_qsort __cdecl
-# endif
-# if (LZO_CC_GNUC || LZO_CC_HIGHC || LZO_CC_NDPC || LZO_CC_PACIFICC || LZO_CC_WATCOMC)
-# elif (LZO_OS_OS2 && (LZO_CC_DMC || LZO_CC_SYMANTECC))
-# define __lzo_cdecl_sighandler __pascal
-# elif (LZO_OS_OS2 && (LZO_CC_ZORTECHC))
-# define __lzo_cdecl_sighandler _stdcall
-# elif (LZO_CC_MSC && (_MSC_VER >= 1400)) && defined(_M_CEE_PURE)
-# define __lzo_cdecl_sighandler __clrcall
-# elif (LZO_CC_MSC && (_MSC_VER >= 600 && _MSC_VER < 700))
-# if defined(_DLL)
-# define __lzo_cdecl_sighandler _far _cdecl _loadds
-# elif defined(_MT)
-# define __lzo_cdecl_sighandler _far _cdecl
-# else
-# define __lzo_cdecl_sighandler _cdecl
-# endif
-# else
-# define __lzo_cdecl_sighandler __cdecl
-# endif
-#elif (LZO_ARCH_I386) && (LZO_CC_WATCOMC)
-# define __lzo_cdecl __cdecl
-#elif (LZO_ARCH_M68K && LZO_OS_TOS && (LZO_CC_PUREC || LZO_CC_TURBOC))
-# define __lzo_cdecl cdecl
-#endif
-#if !defined(__lzo_cdecl)
-# define __lzo_cdecl /*empty*/
-#endif
-#if !defined(__lzo_cdecl_atexit)
-# define __lzo_cdecl_atexit /*empty*/
-#endif
-#if !defined(__lzo_cdecl_main)
-# define __lzo_cdecl_main /*empty*/
-#endif
-#if !defined(__lzo_cdecl_qsort)
-# define __lzo_cdecl_qsort /*empty*/
-#endif
-#if !defined(__lzo_cdecl_sighandler)
-# define __lzo_cdecl_sighandler /*empty*/
-#endif
-#if !defined(__lzo_cdecl_va)
-# define __lzo_cdecl_va __lzo_cdecl
-#endif
-#if !(LZO_CFG_NO_WINDOWS_H)
-#if !defined(LZO_HAVE_WINDOWS_H)
-#if (LZO_OS_CYGWIN || (LZO_OS_EMX && defined(__RSXNT__)) || LZO_OS_WIN32 || LZO_OS_WIN64)
-# if (LZO_CC_WATCOMC && (__WATCOMC__ < 1000))
-# elif ((LZO_OS_WIN32 && defined(__PW32__)) && (LZO_CC_GNUC && (LZO_CC_GNUC < 0x030000ul)))
-# elif ((LZO_OS_CYGWIN || defined(__MINGW32__)) && (LZO_CC_GNUC && (LZO_CC_GNUC < 0x025f00ul)))
-# else
-# define LZO_HAVE_WINDOWS_H 1
-# endif
-#endif
-#endif
-#endif
-#ifndef LZO_SIZEOF_SHORT
-#if defined(SIZEOF_SHORT)
-# define LZO_SIZEOF_SHORT (SIZEOF_SHORT)
-#elif defined(__SIZEOF_SHORT__)
-# define LZO_SIZEOF_SHORT (__SIZEOF_SHORT__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_INT
-#if defined(SIZEOF_INT)
-# define LZO_SIZEOF_INT (SIZEOF_INT)
-#elif defined(__SIZEOF_INT__)
-# define LZO_SIZEOF_INT (__SIZEOF_INT__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_LONG
-#if defined(SIZEOF_LONG)
-# define LZO_SIZEOF_LONG (SIZEOF_LONG)
-#elif defined(__SIZEOF_LONG__)
-# define LZO_SIZEOF_LONG (__SIZEOF_LONG__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_LONG_LONG
-#if defined(SIZEOF_LONG_LONG)
-# define LZO_SIZEOF_LONG_LONG (SIZEOF_LONG_LONG)
-#elif defined(__SIZEOF_LONG_LONG__)
-# define LZO_SIZEOF_LONG_LONG (__SIZEOF_LONG_LONG__)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT16
-#if defined(SIZEOF___INT16)
-# define LZO_SIZEOF___INT16 (SIZEOF___INT16)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT32
-#if defined(SIZEOF___INT32)
-# define LZO_SIZEOF___INT32 (SIZEOF___INT32)
-#endif
-#endif
-#ifndef LZO_SIZEOF___INT64
-#if defined(SIZEOF___INT64)
-# define LZO_SIZEOF___INT64 (SIZEOF___INT64)
-#endif
-#endif
-#ifndef LZO_SIZEOF_VOID_P
-#if defined(SIZEOF_VOID_P)
-# define LZO_SIZEOF_VOID_P (SIZEOF_VOID_P)
-#elif defined(__SIZEOF_POINTER__)
-# define LZO_SIZEOF_VOID_P (__SIZEOF_POINTER__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_SIZE_T
-#if defined(SIZEOF_SIZE_T)
-# define LZO_SIZEOF_SIZE_T (SIZEOF_SIZE_T)
-#elif defined(__SIZEOF_SIZE_T__)
-# define LZO_SIZEOF_SIZE_T (__SIZEOF_SIZE_T__)
-#endif
-#endif
-#ifndef LZO_SIZEOF_PTRDIFF_T
-#if defined(SIZEOF_PTRDIFF_T)
-# define LZO_SIZEOF_PTRDIFF_T (SIZEOF_PTRDIFF_T)
-#elif defined(__SIZEOF_PTRDIFF_T__)
-# define LZO_SIZEOF_PTRDIFF_T (__SIZEOF_PTRDIFF_T__)
-#endif
-#endif
-#define __LZO_LSR(x,b) (((x)+0ul) >> (b))
-#if !defined(LZO_SIZEOF_SHORT)
-# if (LZO_ARCH_CRAY_PVP)
-# define LZO_SIZEOF_SHORT 8
-# elif (USHRT_MAX == LZO_0xffffL)
-# define LZO_SIZEOF_SHORT 2
-# elif (__LZO_LSR(USHRT_MAX,7) == 1)
-# define LZO_SIZEOF_SHORT 1
-# elif (__LZO_LSR(USHRT_MAX,15) == 1)
-# define LZO_SIZEOF_SHORT 2
-# elif (__LZO_LSR(USHRT_MAX,31) == 1)
-# define LZO_SIZEOF_SHORT 4
-# elif (__LZO_LSR(USHRT_MAX,63) == 1)
-# define LZO_SIZEOF_SHORT 8
-# elif (__LZO_LSR(USHRT_MAX,127) == 1)
-# define LZO_SIZEOF_SHORT 16
-# else
-# error "LZO_SIZEOF_SHORT"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_SHORT == sizeof(short))
-#if !defined(LZO_SIZEOF_INT)
-# if (LZO_ARCH_CRAY_PVP)
-# define LZO_SIZEOF_INT 8
-# elif (UINT_MAX == LZO_0xffffL)
-# define LZO_SIZEOF_INT 2
-# elif (UINT_MAX == LZO_0xffffffffL)
-# define LZO_SIZEOF_INT 4
-# elif (__LZO_LSR(UINT_MAX,7) == 1)
-# define LZO_SIZEOF_INT 1
-# elif (__LZO_LSR(UINT_MAX,15) == 1)
-# define LZO_SIZEOF_INT 2
-# elif (__LZO_LSR(UINT_MAX,31) == 1)
-# define LZO_SIZEOF_INT 4
-# elif (__LZO_LSR(UINT_MAX,63) == 1)
-# define LZO_SIZEOF_INT 8
-# elif (__LZO_LSR(UINT_MAX,127) == 1)
-# define LZO_SIZEOF_INT 16
-# else
-# error "LZO_SIZEOF_INT"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_INT == sizeof(int))
-#if !defined(LZO_SIZEOF_LONG)
-# if (ULONG_MAX == LZO_0xffffffffL)
-# define LZO_SIZEOF_LONG 4
-# elif (__LZO_LSR(ULONG_MAX,7) == 1)
-# define LZO_SIZEOF_LONG 1
-# elif (__LZO_LSR(ULONG_MAX,15) == 1)
-# define LZO_SIZEOF_LONG 2
-# elif (__LZO_LSR(ULONG_MAX,31) == 1)
-# define LZO_SIZEOF_LONG 4
-# elif (__LZO_LSR(ULONG_MAX,39) == 1)
-# define LZO_SIZEOF_LONG 5
-# elif (__LZO_LSR(ULONG_MAX,63) == 1)
-# define LZO_SIZEOF_LONG 8
-# elif (__LZO_LSR(ULONG_MAX,127) == 1)
-# define LZO_SIZEOF_LONG 16
-# else
-# error "LZO_SIZEOF_LONG"
-# endif
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_LONG == sizeof(long))
-#if !defined(LZO_SIZEOF_LONG_LONG) && !defined(LZO_SIZEOF___INT64)
-#if (LZO_SIZEOF_LONG > 0 && LZO_SIZEOF_LONG < 8)
-# if defined(__LONG_MAX__) && defined(__LONG_LONG_MAX__)
-# if (LZO_CC_GNUC >= 0x030300ul)
-# if ((__LONG_MAX__-0) == (__LONG_LONG_MAX__-0))
-# define LZO_SIZEOF_LONG_LONG LZO_SIZEOF_LONG
-# elif (__LZO_LSR(__LONG_LONG_MAX__,30) == 1)
-# define LZO_SIZEOF_LONG_LONG 4
-# endif
-# endif
-# endif
-#endif
-#endif
-#if !defined(LZO_SIZEOF_LONG_LONG) && !defined(LZO_SIZEOF___INT64)
-#if (LZO_SIZEOF_LONG > 0 && LZO_SIZEOF_LONG < 8)
-#if (LZO_ARCH_I086 && LZO_CC_DMC)
-#elif (LZO_CC_CILLY) && defined(__GNUC__)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif ((LZO_OS_WIN32 || LZO_OS_WIN64 || defined(_WIN32)) && LZO_CC_MSC && (_MSC_VER >= 1400))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_OS_WIN64 || defined(_WIN64))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_DMC))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_SYMANTECC && (__SC__ >= 0x700)))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_INTELC && defined(__linux__)))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_MWERKS || LZO_CC_PELLESC || LZO_CC_PGI || LZO_CC_SUNPROC))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_ARCH_I386 && (LZO_CC_INTELC || LZO_CC_MSC))
-# define LZO_SIZEOF___INT64 8
-#elif ((LZO_OS_WIN32 || defined(_WIN32)) && (LZO_CC_MSC))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_BORLANDC && (__BORLANDC__ >= 0x0520)))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_ARCH_I386 && (LZO_CC_WATCOMC && (__WATCOMC__ >= 1100)))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_CC_GHS && defined(__LLONG_BIT) && ((__LLONG_BIT-0) == 64))
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_WATCOMC && defined(_INTEGRAL_MAX_BITS) && ((_INTEGRAL_MAX_BITS-0) == 64))
-# define LZO_SIZEOF___INT64 8
-#elif (LZO_OS_OS400 || defined(__OS400__)) && defined(__LLP64_IFC__)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (defined(__vms) || defined(__VMS)) && ((__INITIAL_POINTER_SIZE-0) == 64)
-# define LZO_SIZEOF_LONG_LONG 8
-#elif (LZO_CC_SDCC) && (LZO_SIZEOF_INT == 2)
-#elif 1 && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
-# define LZO_SIZEOF_LONG_LONG 8
-#endif
-#endif
-#endif
-#if defined(__cplusplus) && (LZO_CC_GNUC)
-# if (LZO_CC_GNUC < 0x020800ul)
-# undef LZO_SIZEOF_LONG_LONG
-# endif
-#endif
-#if (LZO_CFG_NO_LONG_LONG)
-# undef LZO_SIZEOF_LONG_LONG
-#elif defined(__NO_LONG_LONG)
-# undef LZO_SIZEOF_LONG_LONG
-#elif defined(_NO_LONGLONG)
-# undef LZO_SIZEOF_LONG_LONG
-#endif
-#if !defined(LZO_WORDSIZE)
-#if (LZO_ARCH_ALPHA)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_AMD64)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_AVR)
-# define LZO_WORDSIZE 1
-#elif (LZO_ARCH_H8300)
-# if defined(__NORMAL_MODE__)
-# define LZO_WORDSIZE 4
-# elif defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_WORDSIZE 4
-# else
-# define LZO_WORDSIZE 2
-# endif
-#elif (LZO_ARCH_I086)
-# define LZO_WORDSIZE 2
-#elif (LZO_ARCH_IA64)
-# define LZO_WORDSIZE 8
-#elif (LZO_ARCH_M16C)
-# define LZO_WORDSIZE 2
-#elif (LZO_ARCH_SPU)
-# define LZO_WORDSIZE 4
-#elif (LZO_ARCH_Z80)
-# define LZO_WORDSIZE 1
-#elif (LZO_SIZEOF_LONG == 8) && ((defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__))
-# define LZO_WORDSIZE 8
-#elif (LZO_OS_OS400 || defined(__OS400__))
-# define LZO_WORDSIZE 8
-#elif (defined(__vms) || defined(__VMS)) && (__INITIAL_POINTER_SIZE+0 == 64)
-# define LZO_WORDSIZE 8
-#endif
-#endif
-#if !defined(LZO_SIZEOF_VOID_P)
-#if defined(__ILP32__) || defined(__ILP32) || defined(_ILP32)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(int) == 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 4)
-# define LZO_SIZEOF_VOID_P 4
-#elif defined(__ILP64__) || defined(__ILP64) || defined(_ILP64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(int) == 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 8)
-# define LZO_SIZEOF_VOID_P 8
-#elif defined(__LLP64__) || defined(__LLP64) || defined(_LLP64) || defined(_WIN64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 4)
-# define LZO_SIZEOF_VOID_P 8
-#elif defined(__LP64__) || defined(__LP64) || defined(_LP64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(long) == 8)
-# define LZO_SIZEOF_VOID_P 8
-#elif (LZO_ARCH_AVR)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_ARCH_C166 || LZO_ARCH_MCS51 || LZO_ARCH_MCS251 || LZO_ARCH_MSP430)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_ARCH_H8300)
-# if defined(__NORMAL_MODE__)
-# define LZO_SIZEOF_VOID_P 2
-# elif defined(__H8300H__) || defined(__H8300S__) || defined(__H8300SX__)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# define LZO_SIZEOF_VOID_P 2
-# endif
-# if (LZO_CC_GNUC && (LZO_CC_GNUC < 0x040000ul)) && (LZO_SIZEOF_INT == 4)
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_INT
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_INT
-# endif
-#elif (LZO_ARCH_I086)
-# if (LZO_MM_TINY || LZO_MM_SMALL || LZO_MM_MEDIUM)
-# define LZO_SIZEOF_VOID_P 2
-# elif (LZO_MM_COMPACT || LZO_MM_LARGE || LZO_MM_HUGE)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# error "invalid LZO_ARCH_I086 memory model"
-# endif
-#elif (LZO_ARCH_M16C)
-# if defined(__m32c_cpu__) || defined(__m32cm_cpu__)
-# define LZO_SIZEOF_VOID_P 4
-# else
-# define LZO_SIZEOF_VOID_P 2
-# endif
-#elif (LZO_ARCH_SPU)
-# define LZO_SIZEOF_VOID_P 4
-#elif (LZO_ARCH_Z80)
-# define LZO_SIZEOF_VOID_P 2
-#elif (LZO_SIZEOF_LONG == 8) && ((defined(__mips__) && defined(__R5900__)) || defined(__MIPS_PSX2__))
-# define LZO_SIZEOF_VOID_P 4
-#elif (LZO_OS_OS400 || defined(__OS400__))
-# if defined(__LLP64_IFC__)
-# define LZO_SIZEOF_VOID_P 8
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-# else
-# define LZO_SIZEOF_VOID_P 16
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-# endif
-#elif (defined(__vms) || defined(__VMS)) && (__INITIAL_POINTER_SIZE+0 == 64)
-# define LZO_SIZEOF_VOID_P 8
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_LONG
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_LONG
-#endif
-#endif
-#if !defined(LZO_SIZEOF_VOID_P)
-# define LZO_SIZEOF_VOID_P LZO_SIZEOF_LONG
-#endif
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_VOID_P == sizeof(void *))
-#if !defined(LZO_SIZEOF_SIZE_T)
-#if (LZO_ARCH_I086 || LZO_ARCH_M16C)
-# define LZO_SIZEOF_SIZE_T 2
-#endif
-#endif
-#if !defined(LZO_SIZEOF_SIZE_T)
-# define LZO_SIZEOF_SIZE_T LZO_SIZEOF_VOID_P
-#endif
-#if defined(offsetof)
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_SIZE_T == sizeof(size_t))
-#endif
-#if !defined(LZO_SIZEOF_PTRDIFF_T)
-#if (LZO_ARCH_I086)
-# if (LZO_MM_TINY || LZO_MM_SMALL || LZO_MM_MEDIUM || LZO_MM_HUGE)
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_VOID_P
-# elif (LZO_MM_COMPACT || LZO_MM_LARGE)
-# if (LZO_CC_BORLANDC || LZO_CC_TURBOC)
-# define LZO_SIZEOF_PTRDIFF_T 4
-# else
-# define LZO_SIZEOF_PTRDIFF_T 2
-# endif
-# else
-# error "invalid LZO_ARCH_I086 memory model"
-# endif
-#endif
-#endif
-#if !defined(LZO_SIZEOF_PTRDIFF_T)
-# define LZO_SIZEOF_PTRDIFF_T LZO_SIZEOF_SIZE_T
-#endif
-#if defined(offsetof)
-LZO_COMPILE_TIME_ASSERT_HEADER(LZO_SIZEOF_PTRDIFF_T == sizeof(ptrdiff_t))
-#endif
-#if !defined(LZO_WORDSIZE)
-# define LZO_WORDSIZE LZO_SIZEOF_VOID_P
-#endif
-#if (LZO_ABI_NEUTRAL_ENDIAN)
-# undef LZO_ABI_BIG_ENDIAN
-# undef LZO_ABI_LITTLE_ENDIAN
-#elif !(LZO_ABI_BIG_ENDIAN) && !(LZO_ABI_LITTLE_ENDIAN)
-#if (LZO_ARCH_ALPHA) && (LZO_ARCH_CRAY_MPP)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif (LZO_ARCH_IA64) && (LZO_OS_POSIX_LINUX || LZO_OS_WIN64)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif (LZO_ARCH_ALPHA || LZO_ARCH_AMD64 || LZO_ARCH_BLACKFIN || LZO_ARCH_CRIS || LZO_ARCH_I086 || LZO_ARCH_I386 || LZO_ARCH_MSP430)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif (LZO_ARCH_AVR32 || LZO_ARCH_M68K || LZO_ARCH_S390 || LZO_ARCH_SPU)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && defined(__IAR_SYSTEMS_ICC__) && defined(__LITTLE_ENDIAN__)
-# if (__LITTLE_ENDIAN__ == 1)
-# define LZO_ABI_LITTLE_ENDIAN 1
-# else
-# define LZO_ABI_BIG_ENDIAN 1
-# endif
-#elif 1 && defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARM_BIG_ENDIAN) && ((__ARM_BIG_ENDIAN)+0)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARMEB__) && !defined(__ARMEL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM) && defined(__ARMEL__) && !defined(__ARMEB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM && LZO_CC_ARMCC_ARMCC)
-# if defined(__BIG_ENDIAN) && defined(__LITTLE_ENDIAN)
-# error "unexpected configuration - check your compiler defines"
-# elif defined(__BIG_ENDIAN)
-# define LZO_ABI_BIG_ENDIAN 1
-# else
-# define LZO_ABI_LITTLE_ENDIAN 1
-# endif
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__ARM_BIG_ENDIAN) && ((__ARM_BIG_ENDIAN)+0)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__AARCH64EB__) && !defined(__AARCH64EL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_ARM64) && defined(__AARCH64EL__) && !defined(__AARCH64EB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#elif 1 && (LZO_ARCH_MIPS) && defined(__MIPSEB__) && !defined(__MIPSEL__)
-# define LZO_ABI_BIG_ENDIAN 1
-#elif 1 && (LZO_ARCH_MIPS) && defined(__MIPSEL__) && !defined(__MIPSEB__)
-# define LZO_ABI_LITTLE_ENDIAN 1
-#endif
-#endif
-#if (LZO_ABI_BIG_ENDIAN) && (LZO_ABI_LITTLE_ENDIAN)
-# error "unexpected configuration - check your compiler defines"
-#endif
-#if (LZO_ABI_BIG_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "be"
-#elif (LZO_ABI_LITTLE_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "le"
-#elif (LZO_ABI_NEUTRAL_ENDIAN)
-# define LZO_INFO_ABI_ENDIAN "neutral"
-#endif
-#if (LZO_SIZEOF_INT == 1 && LZO_SIZEOF_LONG == 2 && LZO_SIZEOF_VOID_P == 2)
-# define LZO_ABI_I8LP16 1
-# define LZO_INFO_ABI_PM "i8lp16"
-#elif (LZO_SIZEOF_INT == 2 && LZO_SIZEOF_LONG == 2 && LZO_SIZEOF_VOID_P == 2)
-# define LZO_ABI_ILP16 1
-# define LZO_INFO_ABI_PM "ilp16"
-#elif (LZO_SIZEOF_INT == 2 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_LP32 1
-# define LZO_INFO_ABI_PM "lp32"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_ILP32 1
-# define LZO_INFO_ABI_PM "ilp32"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 4 && LZO_SIZEOF_VOID_P == 8 && LZO_SIZEOF_SIZE_T == 8)
-# define LZO_ABI_LLP64 1
-# define LZO_INFO_ABI_PM "llp64"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 8)
-# define LZO_ABI_LP64 1
-# define LZO_INFO_ABI_PM "lp64"
-#elif (LZO_SIZEOF_INT == 8 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 8)
-# define LZO_ABI_ILP64 1
-# define LZO_INFO_ABI_PM "ilp64"
-#elif (LZO_SIZEOF_INT == 4 && LZO_SIZEOF_LONG == 8 && LZO_SIZEOF_VOID_P == 4)
-# define LZO_ABI_IP32L64 1
-# define LZO_INFO_ABI_PM "ip32l64"
-#endif
-#if 0
-#elif !defined(__LZO_LIBC_OVERRIDE)
-#if (LZO_LIBC_NAKED)
-# define LZO_INFO_LIBC "naked"
-#elif (LZO_LIBC_FREESTANDING)
-# define LZO_INFO_LIBC "freestanding"
-#elif (LZO_LIBC_MOSTLY_FREESTANDING)
-# define LZO_INFO_LIBC "mfreestanding"
-#elif (LZO_LIBC_ISOC90)
-# define LZO_INFO_LIBC "isoc90"
-#elif (LZO_LIBC_ISOC99)
-# define LZO_INFO_LIBC "isoc99"
-#elif (LZO_CC_ARMCC_ARMCC) && defined(__ARMCLIB_VERSION)
-# define LZO_LIBC_ISOC90 1
-# define LZO_INFO_LIBC "isoc90"
-#elif defined(__dietlibc__)
-# define LZO_LIBC_DIETLIBC 1
-# define LZO_INFO_LIBC "dietlibc"
-#elif defined(_NEWLIB_VERSION)
-# define LZO_LIBC_NEWLIB 1
-# define LZO_INFO_LIBC "newlib"
-#elif defined(__UCLIBC__) && defined(__UCLIBC_MAJOR__) && defined(__UCLIBC_MINOR__)
-# if defined(__UCLIBC_SUBLEVEL__)
-# define LZO_LIBC_UCLIBC (__UCLIBC_MAJOR__ * 0x10000L + (__UCLIBC_MINOR__-0) * 0x100 + (__UCLIBC_SUBLEVEL__-0))
-# else
-# define LZO_LIBC_UCLIBC 0x00090bL
-# endif
-# define LZO_INFO_LIBC "uc" "libc"
-#elif defined(__GLIBC__) && defined(__GLIBC_MINOR__)
-# define LZO_LIBC_GLIBC (__GLIBC__ * 0x10000L + (__GLIBC_MINOR__-0) * 0x100)
-# define LZO_INFO_LIBC "glibc"
-#elif (LZO_CC_MWERKS) && defined(__MSL__)
-# define LZO_LIBC_MSL __MSL__
-# define LZO_INFO_LIBC "msl"
-#elif 1 && defined(__IAR_SYSTEMS_ICC__)
-# define LZO_LIBC_ISOC90 1
-# define LZO_INFO_LIBC "isoc90"
-#else
-# define LZO_LIBC_DEFAULT 1
-# define LZO_INFO_LIBC "default"
-#endif
-#endif
-#if (LZO_ARCH_I386 && (LZO_OS_DOS32 || LZO_OS_WIN32) && (LZO_CC_DMC || LZO_CC_INTELC || LZO_CC_MSC || LZO_CC_PELLESC))
-# define LZO_ASM_SYNTAX_MSC 1
-#elif (LZO_OS_WIN64 && (LZO_CC_DMC || LZO_CC_INTELC || LZO_CC_MSC || LZO_CC_PELLESC))
-#elif (LZO_ARCH_I386 && LZO_CC_GNUC && (LZO_CC_GNUC == 0x011f00ul))
-#elif (LZO_ARCH_I386 && (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC || LZO_CC_PATHSCALE))
-# define LZO_ASM_SYNTAX_GNUC 1
-#elif (LZO_ARCH_AMD64 && (LZO_CC_CLANG || LZO_CC_GNUC || LZO_CC_INTELC || LZO_CC_PATHSCALE))
-# define LZO_ASM_SYNTAX_GNUC 1
-#elif (LZO_CC_GNUC)
-# define LZO_ASM_SYNTAX_GNUC 1
-#endif
-#if (LZO_ASM_SYNTAX_GNUC)
-#if (LZO_ARCH_I386 && LZO_CC_GNUC && (LZO_CC_GNUC < 0x020000ul))
-# define __LZO_ASM_CLOBBER "ax"
-# define __LZO_ASM_CLOBBER_LIST_CC /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#elif (LZO_CC_INTELC && (__INTEL_COMPILER < 1000))
-# define __LZO_ASM_CLOBBER "memory"
-# define __LZO_ASM_CLOBBER_LIST_CC /*empty*/
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY : "memory"
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#else
-# define __LZO_ASM_CLOBBER "cc", "memory"
-# define __LZO_ASM_CLOBBER_LIST_CC : "cc"
-# define __LZO_ASM_CLOBBER_LIST_CC_MEMORY : "cc", "memory"
-# define __LZO_ASM_CLOBBER_LIST_EMPTY /*empty*/
-#endif
-#endif
-#if (LZO_ARCH_ALPHA)
-# define LZO_OPT_AVOID_UINT_INDEX 1
-#elif (LZO_ARCH_AMD64)
-# define LZO_OPT_AVOID_INT_INDEX 1
-# define LZO_OPT_AVOID_UINT_INDEX 1
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-#elif (LZO_ARCH_ARM)
-# if defined(__ARM_FEATURE_UNALIGNED)
-# if ((__ARM_FEATURE_UNALIGNED)+0)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-# elif 1 && (LZO_ARCH_ARM_THUMB2)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# elif 1 && defined(__TARGET_ARCH_ARM) && ((__TARGET_ARCH_ARM)+0 >= 7)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# elif 1 && defined(__TARGET_ARCH_ARM) && ((__TARGET_ARCH_ARM)+0 >= 6) && (defined(__TARGET_PROFILE_A) || defined(__TARGET_PROFILE_R))
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-#elif (LZO_ARCH_ARM64)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-#elif (LZO_ARCH_CRIS)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-#elif (LZO_ARCH_I386)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-#elif (LZO_ARCH_IA64)
-# define LZO_OPT_AVOID_INT_INDEX 1
-# define LZO_OPT_AVOID_UINT_INDEX 1
-# define LZO_OPT_PREFER_POSTINC 1
-#elif (LZO_ARCH_M68K)
-# define LZO_OPT_PREFER_POSTINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-# if defined(__mc68020__) && !defined(__mcoldfire__)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# endif
-#elif (LZO_ARCH_MIPS)
-# define LZO_OPT_AVOID_UINT_INDEX 1
-#elif (LZO_ARCH_POWERPC)
-# define LZO_OPT_PREFER_PREINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-# if (LZO_ABI_BIG_ENDIAN)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# if (LZO_WORDSIZE == 8)
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-# endif
-# endif
-#elif (LZO_ARCH_S390)
-# ifndef LZO_OPT_UNALIGNED16
-# define LZO_OPT_UNALIGNED16 1
-# endif
-# ifndef LZO_OPT_UNALIGNED32
-# define LZO_OPT_UNALIGNED32 1
-# endif
-# if (LZO_WORDSIZE == 8)
-# ifndef LZO_OPT_UNALIGNED64
-# define LZO_OPT_UNALIGNED64 1
-# endif
-# endif
-#elif (LZO_ARCH_SH)
-# define LZO_OPT_PREFER_POSTINC 1
-# define LZO_OPT_PREFER_PREDEC 1
-#endif
-#ifndef LZO_CFG_NO_INLINE_ASM
-#if (LZO_ABI_NEUTRAL_ENDIAN) || (LZO_ARCH_GENERIC)
-# define LZO_CFG_NO_INLINE_ASM 1
-#elif (LZO_CC_LLVM)
-# define LZO_CFG_NO_INLINE_ASM 1
-#endif
-#endif
-#if (LZO_CFG_NO_INLINE_ASM)
-# undef LZO_ASM_SYNTAX_MSC
-# undef LZO_ASM_SYNTAX_GNUC
-# undef __LZO_ASM_CLOBBER
-# undef __LZO_ASM_CLOBBER_LIST_CC
-# undef __LZO_ASM_CLOBBER_LIST_CC_MEMORY
-# undef __LZO_ASM_CLOBBER_LIST_EMPTY
-#endif
-#ifndef LZO_CFG_NO_UNALIGNED
-#if (LZO_ABI_NEUTRAL_ENDIAN) || (LZO_ARCH_GENERIC)
-# define LZO_CFG_NO_UNALIGNED 1
-#endif
-#endif
-#if (LZO_CFG_NO_UNALIGNED)
-# undef LZO_OPT_UNALIGNED16
-# undef LZO_OPT_UNALIGNED32
-# undef LZO_OPT_UNALIGNED64
-#endif
-#if defined(__LZO_INFOSTR_MM)
-#elif (LZO_MM_FLAT) && (defined(__LZO_INFOSTR_PM) || defined(LZO_INFO_ABI_PM))
-# define __LZO_INFOSTR_MM ""
-#elif defined(LZO_INFO_MM)
-# define __LZO_INFOSTR_MM "." LZO_INFO_MM
-#else
-# define __LZO_INFOSTR_MM ""
-#endif
-#if defined(__LZO_INFOSTR_PM)
-#elif defined(LZO_INFO_ABI_PM)
-# define __LZO_INFOSTR_PM "." LZO_INFO_ABI_PM
-#else
-# define __LZO_INFOSTR_PM ""
-#endif
-#if defined(__LZO_INFOSTR_ENDIAN)
-#elif defined(LZO_INFO_ABI_ENDIAN)
-# define __LZO_INFOSTR_ENDIAN "." LZO_INFO_ABI_ENDIAN
-#else
-# define __LZO_INFOSTR_ENDIAN ""
-#endif
-#if defined(__LZO_INFOSTR_OSNAME)
-#elif defined(LZO_INFO_OS_CONSOLE)
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS "." LZO_INFO_OS_CONSOLE
-#elif defined(LZO_INFO_OS_POSIX)
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS "." LZO_INFO_OS_POSIX
-#else
-# define __LZO_INFOSTR_OSNAME LZO_INFO_OS
-#endif
-#if defined(__LZO_INFOSTR_LIBC)
-#elif defined(LZO_INFO_LIBC)
-# define __LZO_INFOSTR_LIBC "." LZO_INFO_LIBC
-#else
-# define __LZO_INFOSTR_LIBC ""
-#endif
-#if defined(__LZO_INFOSTR_CCVER)
-#elif defined(LZO_INFO_CCVER)
-# define __LZO_INFOSTR_CCVER " " LZO_INFO_CCVER
-#else
-# define __LZO_INFOSTR_CCVER ""
-#endif
-#define LZO_INFO_STRING \
- LZO_INFO_ARCH __LZO_INFOSTR_MM __LZO_INFOSTR_PM __LZO_INFOSTR_ENDIAN \
- " " __LZO_INFOSTR_OSNAME __LZO_INFOSTR_LIBC " " LZO_INFO_CC __LZO_INFOSTR_CCVER
-#if !(LZO_CFG_SKIP_LZO_TYPES)
-#if (!(LZO_SIZEOF_SHORT+0 > 0 && LZO_SIZEOF_INT+0 > 0 && LZO_SIZEOF_LONG+0 > 0))
-# error "missing defines for sizes"
-#endif
-#if (!(LZO_SIZEOF_PTRDIFF_T+0 > 0 && LZO_SIZEOF_SIZE_T+0 > 0 && LZO_SIZEOF_VOID_P+0 > 0))
-# error "missing defines for sizes"
-#endif
-#define LZO_TYPEOF_CHAR 1u
-#define LZO_TYPEOF_SHORT 2u
-#define LZO_TYPEOF_INT 3u
-#define LZO_TYPEOF_LONG 4u
-#define LZO_TYPEOF_LONG_LONG 5u
-#define LZO_TYPEOF___INT8 17u
-#define LZO_TYPEOF___INT16 18u
-#define LZO_TYPEOF___INT32 19u
-#define LZO_TYPEOF___INT64 20u
-#define LZO_TYPEOF___INT128 21u
-#define LZO_TYPEOF___INT256 22u
-#define LZO_TYPEOF___MODE_QI 33u
-#define LZO_TYPEOF___MODE_HI 34u
-#define LZO_TYPEOF___MODE_SI 35u
-#define LZO_TYPEOF___MODE_DI 36u
-#define LZO_TYPEOF___MODE_TI 37u
-#define LZO_TYPEOF_CHAR_P 129u
-#if !defined(lzo_llong_t)
-#if (LZO_SIZEOF_LONG_LONG+0 > 0)
-__lzo_gnuc_extension__ typedef long long lzo_llong_t__;
-__lzo_gnuc_extension__ typedef unsigned long long lzo_ullong_t__;
-# define lzo_llong_t lzo_llong_t__
-# define lzo_ullong_t lzo_ullong_t__
-#endif
-#endif
-#if !defined(lzo_int16e_t)
-#if (LZO_SIZEOF_LONG == 2)
-# define lzo_int16e_t long
-# define lzo_uint16e_t unsigned long
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_INT == 2)
-# define lzo_int16e_t int
-# define lzo_uint16e_t unsigned int
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == 2)
-# define lzo_int16e_t short int
-# define lzo_uint16e_t unsigned short int
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF_SHORT
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_HI) && (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x025f00ul) || LZO_CC_LLVM)
- typedef int lzo_int16e_hi_t__ __attribute__((__mode__(__HI__)));
- typedef unsigned int lzo_uint16e_hi_t__ __attribute__((__mode__(__HI__)));
-# define lzo_int16e_t lzo_int16e_hi_t__
-# define lzo_uint16e_t lzo_uint16e_hi_t__
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF___MODE_HI
-#elif (LZO_SIZEOF___INT16 == 2)
-# define lzo_int16e_t __int16
-# define lzo_uint16e_t unsigned __int16
-# define LZO_TYPEOF_LZO_INT16E_T LZO_TYPEOF___INT16
-#else
-#endif
-#endif
-#if defined(lzo_int16e_t)
-# define LZO_SIZEOF_LZO_INT16E_T 2
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16e_t) == 2)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16e_t) == LZO_SIZEOF_LZO_INT16E_T)
-#endif
-#if !defined(lzo_int32e_t)
-#if (LZO_SIZEOF_LONG == 4)
-# define lzo_int32e_t long int
-# define lzo_uint32e_t unsigned long int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_INT == 4)
-# define lzo_int32e_t int
-# define lzo_uint32e_t unsigned int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == 4)
-# define lzo_int32e_t short int
-# define lzo_uint32e_t unsigned short int
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_SHORT
-#elif (LZO_SIZEOF_LONG_LONG == 4)
-# define lzo_int32e_t lzo_llong_t
-# define lzo_uint32e_t lzo_ullong_t
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF_LONG_LONG
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_SI) && (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x025f00ul) || LZO_CC_LLVM) && (__INT_MAX__+0 > 2147483647L)
- typedef int lzo_int32e_si_t__ __attribute__((__mode__(__SI__)));
- typedef unsigned int lzo_uint32e_si_t__ __attribute__((__mode__(__SI__)));
-# define lzo_int32e_t lzo_int32e_si_t__
-# define lzo_uint32e_t lzo_uint32e_si_t__
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___MODE_SI
-#elif 1 && !(LZO_CFG_TYPE_NO_MODE_SI) && (LZO_CC_GNUC >= 0x025f00ul) && defined(__AVR__) && (__LONG_MAX__+0 == 32767L)
- typedef int lzo_int32e_si_t__ __attribute__((__mode__(__SI__)));
- typedef unsigned int lzo_uint32e_si_t__ __attribute__((__mode__(__SI__)));
-# define lzo_int32e_t lzo_int32e_si_t__
-# define lzo_uint32e_t lzo_uint32e_si_t__
-# define LZO_INT32_C(c) (c##LL)
-# define LZO_UINT32_C(c) (c##ULL)
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___MODE_SI
-#elif (LZO_SIZEOF___INT32 == 4)
-# define lzo_int32e_t __int32
-# define lzo_uint32e_t unsigned __int32
-# define LZO_TYPEOF_LZO_INT32E_T LZO_TYPEOF___INT32
-#else
-#endif
-#endif
-#if defined(lzo_int32e_t)
-# define LZO_SIZEOF_LZO_INT32E_T 4
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32e_t) == 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32e_t) == LZO_SIZEOF_LZO_INT32E_T)
-#endif
-#if !defined(lzo_int64e_t)
-#if (LZO_SIZEOF___INT64 == 8)
-# if (LZO_CC_BORLANDC) && !(LZO_CFG_TYPE_PREFER___INT64)
-# define LZO_CFG_TYPE_PREFER___INT64 1
-# endif
-#endif
-#if (LZO_SIZEOF_INT == 8) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_int64e_t int
-# define lzo_uint64e_t unsigned int
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_LONG == 8)
-# define lzo_int64e_t long int
-# define lzo_uint64e_t unsigned long int
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_LONG_LONG == 8) && !(LZO_CFG_TYPE_PREFER___INT64)
-# define lzo_int64e_t lzo_llong_t
-# define lzo_uint64e_t lzo_ullong_t
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF_LONG_LONG
-# if (LZO_CC_BORLANDC)
-# define LZO_INT64_C(c) ((c) + 0ll)
-# define LZO_UINT64_C(c) ((c) + 0ull)
-# elif 0
-# define LZO_INT64_C(c) (__lzo_gnuc_extension__ (c##LL))
-# define LZO_UINT64_C(c) (__lzo_gnuc_extension__ (c##ULL))
-# else
-# define LZO_INT64_C(c) (c##LL)
-# define LZO_UINT64_C(c) (c##ULL)
-# endif
-#elif (LZO_SIZEOF___INT64 == 8)
-# define lzo_int64e_t __int64
-# define lzo_uint64e_t unsigned __int64
-# define LZO_TYPEOF_LZO_INT64E_T LZO_TYPEOF___INT64
-# if (LZO_CC_BORLANDC)
-# define LZO_INT64_C(c) ((c) + 0i64)
-# define LZO_UINT64_C(c) ((c) + 0ui64)
-# else
-# define LZO_INT64_C(c) (c##i64)
-# define LZO_UINT64_C(c) (c##ui64)
-# endif
-#else
-#endif
-#endif
-#if defined(lzo_int64e_t)
-# define LZO_SIZEOF_LZO_INT64E_T 8
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64e_t) == 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64e_t) == LZO_SIZEOF_LZO_INT64E_T)
-#endif
-#if !defined(lzo_int32l_t)
-#if defined(lzo_int32e_t)
-# define lzo_int32l_t lzo_int32e_t
-# define lzo_uint32l_t lzo_uint32e_t
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_LZO_INT32E_T
-# define LZO_TYPEOF_LZO_INT32L_T LZO_TYPEOF_LZO_INT32E_T
-#elif (LZO_SIZEOF_INT >= 4) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_int32l_t int
-# define lzo_uint32l_t unsigned int
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INT32L_T LZO_SIZEOF_INT
-#elif (LZO_SIZEOF_LONG >= 4)
-# define lzo_int32l_t long int
-# define lzo_uint32l_t unsigned long int
-# define LZO_SIZEOF_LZO_INT32L_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_INT32L_T LZO_SIZEOF_LONG
-#else
-# error "lzo_int32l_t"
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32l_t) >= 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32l_t) == LZO_SIZEOF_LZO_INT32L_T)
-#endif
-#if !defined(lzo_int64l_t)
-#if defined(lzo_int64e_t)
-# define lzo_int64l_t lzo_int64e_t
-# define lzo_uint64l_t lzo_uint64e_t
-# define LZO_SIZEOF_LZO_INT64L_T LZO_SIZEOF_LZO_INT64E_T
-# define LZO_TYPEOF_LZO_INT64L_T LZO_TYPEOF_LZO_INT64E_T
-#else
-#endif
-#endif
-#if defined(lzo_int64l_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64l_t) >= 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64l_t) == LZO_SIZEOF_LZO_INT64L_T)
-#endif
-#if !defined(lzo_int32f_t)
-#if (LZO_SIZEOF_SIZE_T >= 8)
-# define lzo_int32f_t lzo_int64l_t
-# define lzo_uint32f_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INT32F_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INT32F_T LZO_TYPEOF_LZO_INT64L_T
-#else
-# define lzo_int32f_t lzo_int32l_t
-# define lzo_uint32f_t lzo_uint32l_t
-# define LZO_SIZEOF_LZO_INT32F_T LZO_SIZEOF_LZO_INT32L_T
-# define LZO_TYPEOF_LZO_INT32F_T LZO_TYPEOF_LZO_INT32L_T
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32f_t) >= 4)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32f_t) == LZO_SIZEOF_LZO_INT32F_T)
-#endif
-#if !defined(lzo_int64f_t)
-#if defined(lzo_int64l_t)
-# define lzo_int64f_t lzo_int64l_t
-# define lzo_uint64f_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INT64F_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INT64F_T LZO_TYPEOF_LZO_INT64L_T
-#else
-#endif
-#endif
-#if defined(lzo_int64f_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64f_t) >= 8)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64f_t) == LZO_SIZEOF_LZO_INT64F_T)
-#endif
-#if !defined(lzo_intptr_t)
-#if 1 && (LZO_OS_OS400 && (LZO_SIZEOF_VOID_P == 16))
-# define __LZO_INTPTR_T_IS_POINTER 1
- typedef char * lzo_intptr_t;
- typedef char * lzo_uintptr_t;
-# define lzo_intptr_t lzo_intptr_t
-# define lzo_uintptr_t lzo_uintptr_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_VOID_P
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_CHAR_P
-#elif (LZO_CC_MSC && (_MSC_VER >= 1300) && (LZO_SIZEOF_VOID_P == 4) && (LZO_SIZEOF_INT == 4))
- typedef __w64 int lzo_intptr_t;
- typedef __w64 unsigned int lzo_uintptr_t;
-# define lzo_intptr_t lzo_intptr_t
-# define lzo_uintptr_t lzo_uintptr_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_SHORT == LZO_SIZEOF_VOID_P) && (LZO_SIZEOF_INT > LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t short
-# define lzo_uintptr_t unsigned short
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_SHORT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_SHORT
-#elif (LZO_SIZEOF_INT >= LZO_SIZEOF_VOID_P) && (LZO_SIZEOF_INT < LZO_SIZEOF_LONG)
-# define lzo_intptr_t int
-# define lzo_uintptr_t unsigned int
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_INT
-#elif (LZO_SIZEOF_LONG >= LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t long
-# define lzo_uintptr_t unsigned long
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_LONG
-#elif (LZO_SIZEOF_LZO_INT64L_T >= LZO_SIZEOF_VOID_P)
-# define lzo_intptr_t lzo_int64l_t
-# define lzo_uintptr_t lzo_uint64l_t
-# define LZO_SIZEOF_LZO_INTPTR_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_INTPTR_T LZO_TYPEOF_LZO_INT64L_T
-#else
-# error "lzo_intptr_t"
-#endif
-#endif
-#if 1
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_intptr_t) >= sizeof(void *))
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_intptr_t) == sizeof(lzo_uintptr_t))
-#endif
-#if !defined(lzo_word_t)
-#if defined(LZO_WORDSIZE) && (LZO_WORDSIZE+0 > 0)
-#if (LZO_WORDSIZE == LZO_SIZEOF_LZO_INTPTR_T) && !(__LZO_INTPTR_T_IS_POINTER)
-# define lzo_word_t lzo_uintptr_t
-# define lzo_sword_t lzo_intptr_t
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LZO_INTPTR_T
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_LZO_INTPTR_T
-#elif (LZO_WORDSIZE == LZO_SIZEOF_LONG)
-# define lzo_word_t unsigned long
-# define lzo_sword_t long
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LONG
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_LONG
-#elif (LZO_WORDSIZE == LZO_SIZEOF_INT)
-# define lzo_word_t unsigned int
-# define lzo_sword_t int
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_INT
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_INT
-#elif (LZO_WORDSIZE == LZO_SIZEOF_SHORT)
-# define lzo_word_t unsigned short
-# define lzo_sword_t short
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_SHORT
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_SHORT
-#elif (LZO_WORDSIZE == 1)
-# define lzo_word_t unsigned char
-# define lzo_sword_t signed char
-# define LZO_SIZEOF_LZO_WORD_T 1
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF_CHAR
-#elif (LZO_WORDSIZE == LZO_SIZEOF_LZO_INT64L_T)
-# define lzo_word_t lzo_uint64l_t
-# define lzo_sword_t lzo_int64l_t
-# define LZO_SIZEOF_LZO_WORD_T LZO_SIZEOF_LZO_INT64L_T
-# define LZO_TYPEOF_LZO_WORD_T LZO_SIZEOF_LZO_INT64L_T
-#elif (LZO_ARCH_SPU) && (LZO_CC_GNUC)
-#if 0
- typedef unsigned lzo_word_t __attribute__((__mode__(__V16QI__)));
- typedef int lzo_sword_t __attribute__((__mode__(__V16QI__)));
-# define lzo_word_t lzo_word_t
-# define lzo_sword_t lzo_sword_t
-# define LZO_SIZEOF_LZO_WORD_T 16
-# define LZO_TYPEOF_LZO_WORD_T LZO_TYPEOF___MODE_V16QI
-#endif
-#else
-# error "lzo_word_t"
-#endif
-#endif
-#endif
-#if 1 && defined(lzo_word_t)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_word_t) == LZO_WORDSIZE)
- LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_sword_t) == LZO_WORDSIZE)
-#endif
-#if 1
-#define lzo_int8_t signed char
-#define lzo_uint8_t unsigned char
-#define LZO_SIZEOF_LZO_INT8_T 1
-#define LZO_TYPEOF_LZO_INT8_T LZO_TYPEOF_CHAR
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int8_t) == 1)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int8_t) == sizeof(lzo_uint8_t))
-#endif
-#if defined(lzo_int16e_t)
-#define lzo_int16_t lzo_int16e_t
-#define lzo_uint16_t lzo_uint16e_t
-#define LZO_SIZEOF_LZO_INT16_T LZO_SIZEOF_LZO_INT16E_T
-#define LZO_TYPEOF_LZO_INT16_T LZO_TYPEOF_LZO_INT16E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16_t) == 2)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16_t) == sizeof(lzo_uint16_t))
-#endif
-#if defined(lzo_int32e_t)
-#define lzo_int32_t lzo_int32e_t
-#define lzo_uint32_t lzo_uint32e_t
-#define LZO_SIZEOF_LZO_INT32_T LZO_SIZEOF_LZO_INT32E_T
-#define LZO_TYPEOF_LZO_INT32_T LZO_TYPEOF_LZO_INT32E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32_t) == 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32_t) == sizeof(lzo_uint32_t))
-#endif
-#if defined(lzo_int64e_t)
-#define lzo_int64_t lzo_int64e_t
-#define lzo_uint64_t lzo_uint64e_t
-#define LZO_SIZEOF_LZO_INT64_T LZO_SIZEOF_LZO_INT64E_T
-#define LZO_TYPEOF_LZO_INT64_T LZO_TYPEOF_LZO_INT64E_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64_t) == 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64_t) == sizeof(lzo_uint64_t))
-#endif
-#if 1
-#define lzo_int_least32_t lzo_int32l_t
-#define lzo_uint_least32_t lzo_uint32l_t
-#define LZO_SIZEOF_LZO_INT_LEAST32_T LZO_SIZEOF_LZO_INT32L_T
-#define LZO_TYPEOF_LZO_INT_LEAST32_T LZO_TYPEOF_LZO_INT32L_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least32_t) >= 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least32_t) == sizeof(lzo_uint_least32_t))
-#endif
-#if defined(lzo_int64l_t)
-#define lzo_int_least64_t lzo_int64l_t
-#define lzo_uint_least64_t lzo_uint64l_t
-#define LZO_SIZEOF_LZO_INT_LEAST64_T LZO_SIZEOF_LZO_INT64L_T
-#define LZO_TYPEOF_LZO_INT_LEAST64_T LZO_TYPEOF_LZO_INT64L_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least64_t) >= 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_least64_t) == sizeof(lzo_uint_least64_t))
-#endif
-#if 1
-#define lzo_int_fast32_t lzo_int32f_t
-#define lzo_uint_fast32_t lzo_uint32f_t
-#define LZO_SIZEOF_LZO_INT_FAST32_T LZO_SIZEOF_LZO_INT32F_T
-#define LZO_TYPEOF_LZO_INT_FAST32_T LZO_TYPEOF_LZO_INT32F_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast32_t) >= 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast32_t) == sizeof(lzo_uint_fast32_t))
-#endif
-#if defined(lzo_int64f_t)
-#define lzo_int_fast64_t lzo_int64f_t
-#define lzo_uint_fast64_t lzo_uint64f_t
-#define LZO_SIZEOF_LZO_INT_FAST64_T LZO_SIZEOF_LZO_INT64F_T
-#define LZO_TYPEOF_LZO_INT_FAST64_T LZO_TYPEOF_LZO_INT64F_T
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast64_t) >= 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int_fast64_t) == sizeof(lzo_uint_fast64_t))
-#endif
-#if !defined(LZO_INT16_C)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 2)
-# define LZO_INT16_C(c) ((c) + 0)
-# define LZO_UINT16_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 2)
-# define LZO_INT16_C(c) ((c) + 0L)
-# define LZO_UINT16_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 2)
-# define LZO_INT16_C(c) (c)
-# define LZO_UINT16_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 2)
-# define LZO_INT16_C(c) (c##L)
-# define LZO_UINT16_C(c) (c##UL)
-# else
-# error "LZO_INT16_C"
-# endif
-#endif
-#if !defined(LZO_INT32_C)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 4)
-# define LZO_INT32_C(c) ((c) + 0)
-# define LZO_UINT32_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 4)
-# define LZO_INT32_C(c) ((c) + 0L)
-# define LZO_UINT32_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 4)
-# define LZO_INT32_C(c) (c)
-# define LZO_UINT32_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 4)
-# define LZO_INT32_C(c) (c##L)
-# define LZO_UINT32_C(c) (c##UL)
-# elif (LZO_SIZEOF_LONG_LONG >= 4)
-# define LZO_INT32_C(c) (c##LL)
-# define LZO_UINT32_C(c) (c##ULL)
-# else
-# error "LZO_INT32_C"
-# endif
-#endif
-#if !defined(LZO_INT64_C) && defined(lzo_int64l_t)
-# if (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_INT >= 8)
-# define LZO_INT64_C(c) ((c) + 0)
-# define LZO_UINT64_C(c) ((c) + 0U)
-# elif (LZO_BROKEN_INTEGRAL_CONSTANTS) && (LZO_SIZEOF_LONG >= 8)
-# define LZO_INT64_C(c) ((c) + 0L)
-# define LZO_UINT64_C(c) ((c) + 0UL)
-# elif (LZO_SIZEOF_INT >= 8)
-# define LZO_INT64_C(c) (c)
-# define LZO_UINT64_C(c) (c##U)
-# elif (LZO_SIZEOF_LONG >= 8)
-# define LZO_INT64_C(c) (c##L)
-# define LZO_UINT64_C(c) (c##UL)
-# else
-# error "LZO_INT64_C"
-# endif
-#endif
-#endif
-
-#endif
-
-#endif
-
-#undef LZO_HAVE_CONFIG_H
-#include "minilzo.h"
-
-#if !defined(MINILZO_VERSION) || (MINILZO_VERSION != 0x2090)
-# error "version mismatch in miniLZO source files"
-#endif
-
-#ifdef MINILZO_HAVE_CONFIG_H
-# define LZO_HAVE_CONFIG_H 1
-#endif
-
-#ifndef __LZO_CONF_H
-#define __LZO_CONF_H 1
-
-#if !defined(__LZO_IN_MINILZO)
-#if defined(LZO_CFG_FREESTANDING) && (LZO_CFG_FREESTANDING)
-# define LZO_LIBC_FREESTANDING 1
-# define LZO_OS_FREESTANDING 1
-#endif
-#if defined(LZO_CFG_EXTRA_CONFIG_HEADER)
-# include LZO_CFG_EXTRA_CONFIG_HEADER
-#endif
-#if defined(__LZOCONF_H) || defined(__LZOCONF_H_INCLUDED)
-# error "include this file first"
-#endif
-#if defined(LZO_CFG_BUILD_DLL) && (LZO_CFG_BUILD_DLL+0) && !defined(__LZO_EXPORT1) && !defined(__LZO_EXPORT2) && 0
-#ifndef __LZODEFS_H_INCLUDED
-#if defined(LZO_HAVE_CONFIG_H)
-#error # include <config.h>
-#endif
-#include <limits.h>
-#include <stddef.h>
-#include "lzodefs.h"
-#endif
-#endif
-#include "lzoconf.h"
-#if defined(LZO_CFG_EXTRA_CONFIG_HEADER2)
-# include LZO_CFG_EXTRA_CONFIG_HEADER2
-#endif
-#endif
-
-#if !defined(__LZOCONF_H_INCLUDED) || (LZO_VERSION+0 != 0x2090)
-# error "version mismatch"
-#endif
-
-#if (LZO_CC_MSC && (_MSC_VER >= 1000 && _MSC_VER < 1100))
-# pragma warning(disable: 4702)
-#endif
-#if (LZO_CC_MSC && (_MSC_VER >= 1000))
-# pragma warning(disable: 4127 4701)
-# pragma warning(disable: 4514 4710 4711)
-#endif
-#if (LZO_CC_MSC && (_MSC_VER >= 1300))
-# pragma warning(disable: 4820)
-#endif
-#if (LZO_CC_MSC && (_MSC_VER >= 1800))
-# pragma warning(disable: 4746)
-#endif
-#if (LZO_CC_INTELC && (__INTEL_COMPILER >= 900))
-# pragma warning(disable: 1684)
-#endif
-
-#if (LZO_CC_SUNPROC)
-#if !defined(__cplusplus)
-# pragma error_messages(off,E_END_OF_LOOP_CODE_NOT_REACHED)
-# pragma error_messages(off,E_LOOP_NOT_ENTERED_AT_TOP)
-# pragma error_messages(off,E_STATEMENT_NOT_REACHED)
-#endif
-#endif
-
-#if !defined(__LZO_NOEXPORT1)
-# define __LZO_NOEXPORT1 /*empty*/
-#endif
-#if !defined(__LZO_NOEXPORT2)
-# define __LZO_NOEXPORT2 /*empty*/
-#endif
-
-#if 1
-# define LZO_PUBLIC_DECL(r) LZO_EXTERN(r)
-#endif
-#if 1
-# define LZO_PUBLIC_IMPL(r) LZO_PUBLIC(r)
-#endif
-#if !defined(LZO_LOCAL_DECL)
-# define LZO_LOCAL_DECL(r) __LZO_EXTERN_C LZO_LOCAL_IMPL(r)
-#endif
-#if !defined(LZO_LOCAL_IMPL)
-# define LZO_LOCAL_IMPL(r) __LZO_NOEXPORT1 r __LZO_NOEXPORT2 __LZO_CDECL
-#endif
-#if 1
-# define LZO_STATIC_DECL(r) LZO_PRIVATE(r)
-#endif
-#if 1
-# define LZO_STATIC_IMPL(r) LZO_PRIVATE(r)
-#endif
-
-#if defined(__LZO_IN_MINILZO) || (LZO_CFG_FREESTANDING)
-#elif 1
-# include <string.h>
-#else
-# define LZO_WANT_ACC_INCD_H 1
-#endif
-#if defined(LZO_HAVE_CONFIG_H)
-# define LZO_CFG_NO_CONFIG_HEADER 1
-#endif
-
-#if 1 && !defined(LZO_CFG_FREESTANDING)
-#if 1 && !defined(HAVE_STRING_H)
-#define HAVE_STRING_H 1
-#endif
-#if 1 && !defined(HAVE_MEMCMP)
-#define HAVE_MEMCMP 1
-#endif
-#if 1 && !defined(HAVE_MEMCPY)
-#define HAVE_MEMCPY 1
-#endif
-#if 1 && !defined(HAVE_MEMMOVE)
-#define HAVE_MEMMOVE 1
-#endif
-#if 1 && !defined(HAVE_MEMSET)
-#define HAVE_MEMSET 1
-#endif
-#endif
-
-#if 1 && defined(HAVE_STRING_H)
-#include <string.h>
-#endif
-
-#if 1 || defined(lzo_int8_t) || defined(lzo_uint8_t)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int8_t) == 1)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint8_t) == 1)
-#endif
-#if 1 || defined(lzo_int16_t) || defined(lzo_uint16_t)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int16_t) == 2)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint16_t) == 2)
-#endif
-#if 1 || defined(lzo_int32_t) || defined(lzo_uint32_t)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int32_t) == 4)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint32_t) == 4)
-#endif
-#if defined(lzo_int64_t) || defined(lzo_uint64_t)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_int64_t) == 8)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(lzo_uint64_t) == 8)
-#endif
-
-#if (LZO_CFG_FREESTANDING)
-# undef HAVE_MEMCMP
-# undef HAVE_MEMCPY
-# undef HAVE_MEMMOVE
-# undef HAVE_MEMSET
-#endif
-
-#if !(HAVE_MEMCMP)
-# undef memcmp
-# define memcmp(a,b,c) lzo_memcmp(a,b,c)
-#else
-# undef lzo_memcmp
-# define lzo_memcmp(a,b,c) memcmp(a,b,c)
-#endif
-#if !(HAVE_MEMCPY)
-# undef memcpy
-# define memcpy(a,b,c) lzo_memcpy(a,b,c)
-#else
-# undef lzo_memcpy
-# define lzo_memcpy(a,b,c) memcpy(a,b,c)
-#endif
-#if !(HAVE_MEMMOVE)
-# undef memmove
-# define memmove(a,b,c) lzo_memmove(a,b,c)
-#else
-# undef lzo_memmove
-# define lzo_memmove(a,b,c) memmove(a,b,c)
-#endif
-#if !(HAVE_MEMSET)
-# undef memset
-# define memset(a,b,c) lzo_memset(a,b,c)
-#else
-# undef lzo_memset
-# define lzo_memset(a,b,c) memset(a,b,c)
-#endif
-
-#undef NDEBUG
-#if (LZO_CFG_FREESTANDING)
-# undef LZO_DEBUG
-# define NDEBUG 1
-# undef assert
-# define assert(e) ((void)0)
-#else
-# if !defined(LZO_DEBUG)
-# define NDEBUG 1
-# endif
-# include <assert.h>
-#endif
-
-#if 0 && defined(__BOUNDS_CHECKING_ON)
-# include <unchecked.h>
-#else
-# define BOUNDS_CHECKING_OFF_DURING(stmt) stmt
-# define BOUNDS_CHECKING_OFF_IN_EXPR(expr) (expr)
-#endif
-
-#if (LZO_CFG_PGO)
-# undef __lzo_likely
-# undef __lzo_unlikely
-# define __lzo_likely(e) (e)
-# define __lzo_unlikely(e) (e)
-#endif
-
-#undef _
-#undef __
-#undef ___
-#undef ____
-#undef _p0
-#undef _p1
-#undef _p2
-#undef _p3
-#undef _p4
-#undef _s0
-#undef _s1
-#undef _s2
-#undef _s3
-#undef _s4
-#undef _ww
-
-#if 1
-# define LZO_BYTE(x) ((unsigned char) (x))
-#else
-# define LZO_BYTE(x) ((unsigned char) ((x) & 0xff))
-#endif
-
-#define LZO_MAX(a,b) ((a) >= (b) ? (a) : (b))
-#define LZO_MIN(a,b) ((a) <= (b) ? (a) : (b))
-#define LZO_MAX3(a,b,c) ((a) >= (b) ? LZO_MAX(a,c) : LZO_MAX(b,c))
-#define LZO_MIN3(a,b,c) ((a) <= (b) ? LZO_MIN(a,c) : LZO_MIN(b,c))
-
-#define lzo_sizeof(type) ((lzo_uint) (sizeof(type)))
-
-#define LZO_HIGH(array) ((lzo_uint) (sizeof(array)/sizeof(*(array))))
-
-#define LZO_SIZE(bits) (1u << (bits))
-#define LZO_MASK(bits) (LZO_SIZE(bits) - 1)
-
-#define LZO_USIZE(bits) ((lzo_uint) 1 << (bits))
-#define LZO_UMASK(bits) (LZO_USIZE(bits) - 1)
-
-#if !defined(DMUL)
-#if 0
-
-# define DMUL(a,b) ((lzo_xint) ((lzo_uint32_t)(a) * (lzo_uint32_t)(b)))
-#else
-# define DMUL(a,b) ((lzo_xint) ((a) * (b)))
-#endif
-#endif
-
-#ifndef __LZO_FUNC_H
-#define __LZO_FUNC_H 1
-
-#if !defined(LZO_BITOPS_USE_ASM_BITSCAN) && !defined(LZO_BITOPS_USE_GNUC_BITSCAN) && !defined(LZO_BITOPS_USE_MSC_BITSCAN)
-#if 1 && (LZO_ARCH_AMD64) && (LZO_CC_GNUC && (LZO_CC_GNUC < 0x040000ul)) && (LZO_ASM_SYNTAX_GNUC)
-#define LZO_BITOPS_USE_ASM_BITSCAN 1
-#elif (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x030400ul) || (LZO_CC_INTELC_GNUC && (__INTEL_COMPILER >= 1000)) || (LZO_CC_LLVM && (!defined(__llvm_tools_version__) || (__llvm_tools_version__+0 >= 0x010500ul))))
-#define LZO_BITOPS_USE_GNUC_BITSCAN 1
-#elif (LZO_OS_WIN32 || LZO_OS_WIN64) && ((LZO_CC_INTELC_MSC && (__INTEL_COMPILER >= 1010)) || (LZO_CC_MSC && (_MSC_VER >= 1400)))
-#define LZO_BITOPS_USE_MSC_BITSCAN 1
-#if (LZO_CC_MSC) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-#include <intrin.h>
-#endif
-#if (LZO_CC_MSC) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
-#pragma intrinsic(_BitScanReverse)
-#pragma intrinsic(_BitScanForward)
-#endif
-#if (LZO_CC_MSC) && (LZO_ARCH_AMD64)
-#pragma intrinsic(_BitScanReverse64)
-#pragma intrinsic(_BitScanForward64)
-#endif
-#endif
-#endif
-
-__lzo_static_forceinline unsigned lzo_bitops_ctlz32_func(lzo_uint32_t v)
-{
-#if (LZO_BITOPS_USE_MSC_BITSCAN) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
- unsigned long r; (void) _BitScanReverse(&r, v); return (unsigned) r ^ 31;
-#define lzo_bitops_ctlz32(v) lzo_bitops_ctlz32_func(v)
-#elif (LZO_BITOPS_USE_ASM_BITSCAN) && (LZO_ARCH_AMD64 || LZO_ARCH_I386) && (LZO_ASM_SYNTAX_GNUC)
- lzo_uint32_t r;
- __asm__("bsr %1,%0" : "=r" (r) : "rm" (v) __LZO_ASM_CLOBBER_LIST_CC);
- return (unsigned) r ^ 31;
-#define lzo_bitops_ctlz32(v) lzo_bitops_ctlz32_func(v)
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_INT == 4)
- unsigned r; r = (unsigned) __builtin_clz(v); return r;
-#define lzo_bitops_ctlz32(v) ((unsigned) __builtin_clz(v))
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_LONG == 8) && (LZO_WORDSIZE >= 8)
- unsigned r; r = (unsigned) __builtin_clzl(v); return r ^ 32;
-#define lzo_bitops_ctlz32(v) (((unsigned) __builtin_clzl(v)) ^ 32)
-#else
- LZO_UNUSED(v); return 0;
-#endif
-}
-
-#if defined(lzo_uint64_t)
-__lzo_static_forceinline unsigned lzo_bitops_ctlz64_func(lzo_uint64_t v)
-{
-#if (LZO_BITOPS_USE_MSC_BITSCAN) && (LZO_ARCH_AMD64)
- unsigned long r; (void) _BitScanReverse64(&r, v); return (unsigned) r ^ 63;
-#define lzo_bitops_ctlz64(v) lzo_bitops_ctlz64_func(v)
-#elif (LZO_BITOPS_USE_ASM_BITSCAN) && (LZO_ARCH_AMD64) && (LZO_ASM_SYNTAX_GNUC)
- lzo_uint64_t r;
- __asm__("bsr %1,%0" : "=r" (r) : "rm" (v) __LZO_ASM_CLOBBER_LIST_CC);
- return (unsigned) r ^ 63;
-#define lzo_bitops_ctlz64(v) lzo_bitops_ctlz64_func(v)
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_LONG == 8) && (LZO_WORDSIZE >= 8)
- unsigned r; r = (unsigned) __builtin_clzl(v); return r;
-#define lzo_bitops_ctlz64(v) ((unsigned) __builtin_clzl(v))
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_LONG_LONG == 8) && (LZO_WORDSIZE >= 8)
- unsigned r; r = (unsigned) __builtin_clzll(v); return r;
-#define lzo_bitops_ctlz64(v) ((unsigned) __builtin_clzll(v))
-#else
- LZO_UNUSED(v); return 0;
-#endif
-}
-#endif
-
-__lzo_static_forceinline unsigned lzo_bitops_cttz32_func(lzo_uint32_t v)
-{
-#if (LZO_BITOPS_USE_MSC_BITSCAN) && (LZO_ARCH_AMD64 || LZO_ARCH_I386)
- unsigned long r; (void) _BitScanForward(&r, v); return (unsigned) r;
-#define lzo_bitops_cttz32(v) lzo_bitops_cttz32_func(v)
-#elif (LZO_BITOPS_USE_ASM_BITSCAN) && (LZO_ARCH_AMD64 || LZO_ARCH_I386) && (LZO_ASM_SYNTAX_GNUC)
- lzo_uint32_t r;
- __asm__("bsf %1,%0" : "=r" (r) : "rm" (v) __LZO_ASM_CLOBBER_LIST_CC);
- return (unsigned) r;
-#define lzo_bitops_cttz32(v) lzo_bitops_cttz32_func(v)
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_INT >= 4)
- unsigned r; r = (unsigned) __builtin_ctz(v); return r;
-#define lzo_bitops_cttz32(v) ((unsigned) __builtin_ctz(v))
-#else
- LZO_UNUSED(v); return 0;
-#endif
-}
-
-#if defined(lzo_uint64_t)
-__lzo_static_forceinline unsigned lzo_bitops_cttz64_func(lzo_uint64_t v)
-{
-#if (LZO_BITOPS_USE_MSC_BITSCAN) && (LZO_ARCH_AMD64)
- unsigned long r; (void) _BitScanForward64(&r, v); return (unsigned) r;
-#define lzo_bitops_cttz64(v) lzo_bitops_cttz64_func(v)
-#elif (LZO_BITOPS_USE_ASM_BITSCAN) && (LZO_ARCH_AMD64) && (LZO_ASM_SYNTAX_GNUC)
- lzo_uint64_t r;
- __asm__("bsf %1,%0" : "=r" (r) : "rm" (v) __LZO_ASM_CLOBBER_LIST_CC);
- return (unsigned) r;
-#define lzo_bitops_cttz64(v) lzo_bitops_cttz64_func(v)
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_LONG >= 8) && (LZO_WORDSIZE >= 8)
- unsigned r; r = (unsigned) __builtin_ctzl(v); return r;
-#define lzo_bitops_cttz64(v) ((unsigned) __builtin_ctzl(v))
-#elif (LZO_BITOPS_USE_GNUC_BITSCAN) && (LZO_SIZEOF_LONG_LONG >= 8) && (LZO_WORDSIZE >= 8)
- unsigned r; r = (unsigned) __builtin_ctzll(v); return r;
-#define lzo_bitops_cttz64(v) ((unsigned) __builtin_ctzll(v))
-#else
- LZO_UNUSED(v); return 0;
-#endif
-}
-#endif
-
-lzo_unused_funcs_impl(void, lzo_bitops_unused_funcs)(void)
-{
- LZO_UNUSED_FUNC(lzo_bitops_unused_funcs);
- LZO_UNUSED_FUNC(lzo_bitops_ctlz32_func);
- LZO_UNUSED_FUNC(lzo_bitops_cttz32_func);
-#if defined(lzo_uint64_t)
- LZO_UNUSED_FUNC(lzo_bitops_ctlz64_func);
- LZO_UNUSED_FUNC(lzo_bitops_cttz64_func);
-#endif
-}
-
-#if defined(__lzo_alignof) && !(LZO_CFG_NO_UNALIGNED)
-#if !defined(lzo_memops_tcheck__) && 0
-#define lzo_memops_tcheck__(t,a,b) ((void)0, sizeof(t) == (a) && __lzo_alignof(t) == (b))
-#endif
-#endif
-#ifndef lzo_memops_TU0p
-#define lzo_memops_TU0p void __LZO_MMODEL *
-#endif
-#ifndef lzo_memops_TU1p
-#define lzo_memops_TU1p unsigned char __LZO_MMODEL *
-#endif
-#ifndef lzo_memops_TU2p
-#if (LZO_OPT_UNALIGNED16)
-typedef lzo_uint16_t __lzo_may_alias lzo_memops_TU2;
-#define lzo_memops_TU2p volatile lzo_memops_TU2 *
-#elif defined(__lzo_byte_struct)
-__lzo_byte_struct(lzo_memops_TU2_struct,2)
-typedef struct lzo_memops_TU2_struct lzo_memops_TU2;
-#else
-struct lzo_memops_TU2_struct { unsigned char a[2]; } __lzo_may_alias;
-typedef struct lzo_memops_TU2_struct lzo_memops_TU2;
-#endif
-#ifndef lzo_memops_TU2p
-#define lzo_memops_TU2p lzo_memops_TU2 *
-#endif
-#endif
-#ifndef lzo_memops_TU4p
-#if (LZO_OPT_UNALIGNED32)
-typedef lzo_uint32_t __lzo_may_alias lzo_memops_TU4;
-#define lzo_memops_TU4p volatile lzo_memops_TU4 __LZO_MMODEL *
-#elif defined(__lzo_byte_struct)
-__lzo_byte_struct(lzo_memops_TU4_struct,4)
-typedef struct lzo_memops_TU4_struct lzo_memops_TU4;
-#else
-struct lzo_memops_TU4_struct { unsigned char a[4]; } __lzo_may_alias;
-typedef struct lzo_memops_TU4_struct lzo_memops_TU4;
-#endif
-#ifndef lzo_memops_TU4p
-#define lzo_memops_TU4p lzo_memops_TU4 __LZO_MMODEL *
-#endif
-#endif
-#ifndef lzo_memops_TU8p
-#if (LZO_OPT_UNALIGNED64)
-typedef lzo_uint64_t __lzo_may_alias lzo_memops_TU8;
-#define lzo_memops_TU8p volatile lzo_memops_TU8 __LZO_MMODEL *
-#elif defined(__lzo_byte_struct)
-__lzo_byte_struct(lzo_memops_TU8_struct,8)
-typedef struct lzo_memops_TU8_struct lzo_memops_TU8;
-#else
-struct lzo_memops_TU8_struct { unsigned char a[8]; } __lzo_may_alias;
-typedef struct lzo_memops_TU8_struct lzo_memops_TU8;
-#endif
-#ifndef lzo_memops_TU8p
-#define lzo_memops_TU8p lzo_memops_TU8 __LZO_MMODEL *
-#endif
-#endif
-#ifndef lzo_memops_set_TU1p
-#define lzo_memops_set_TU1p volatile lzo_memops_TU1p
-#endif
-#ifndef lzo_memops_move_TU1p
-#define lzo_memops_move_TU1p lzo_memops_TU1p
-#endif
-#define LZO_MEMOPS_SET1(dd,cc) \
- LZO_BLOCK_BEGIN \
- lzo_memops_set_TU1p d__1 = (lzo_memops_set_TU1p) (lzo_memops_TU0p) (dd); \
- d__1[0] = LZO_BYTE(cc); \
- LZO_BLOCK_END
-#define LZO_MEMOPS_SET2(dd,cc) \
- LZO_BLOCK_BEGIN \
- lzo_memops_set_TU1p d__2 = (lzo_memops_set_TU1p) (lzo_memops_TU0p) (dd); \
- d__2[0] = LZO_BYTE(cc); d__2[1] = LZO_BYTE(cc); \
- LZO_BLOCK_END
-#define LZO_MEMOPS_SET3(dd,cc) \
- LZO_BLOCK_BEGIN \
- lzo_memops_set_TU1p d__3 = (lzo_memops_set_TU1p) (lzo_memops_TU0p) (dd); \
- d__3[0] = LZO_BYTE(cc); d__3[1] = LZO_BYTE(cc); d__3[2] = LZO_BYTE(cc); \
- LZO_BLOCK_END
-#define LZO_MEMOPS_SET4(dd,cc) \
- LZO_BLOCK_BEGIN \
- lzo_memops_set_TU1p d__4 = (lzo_memops_set_TU1p) (lzo_memops_TU0p) (dd); \
- d__4[0] = LZO_BYTE(cc); d__4[1] = LZO_BYTE(cc); d__4[2] = LZO_BYTE(cc); d__4[3] = LZO_BYTE(cc); \
- LZO_BLOCK_END
-#define LZO_MEMOPS_MOVE1(dd,ss) \
- LZO_BLOCK_BEGIN \
- lzo_memops_move_TU1p d__1 = (lzo_memops_move_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_move_TU1p s__1 = (const lzo_memops_move_TU1p) (const lzo_memops_TU0p) (ss); \
- d__1[0] = s__1[0]; \
- LZO_BLOCK_END
-#define LZO_MEMOPS_MOVE2(dd,ss) \
- LZO_BLOCK_BEGIN \
- lzo_memops_move_TU1p d__2 = (lzo_memops_move_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_move_TU1p s__2 = (const lzo_memops_move_TU1p) (const lzo_memops_TU0p) (ss); \
- d__2[0] = s__2[0]; d__2[1] = s__2[1]; \
- LZO_BLOCK_END
-#define LZO_MEMOPS_MOVE3(dd,ss) \
- LZO_BLOCK_BEGIN \
- lzo_memops_move_TU1p d__3 = (lzo_memops_move_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_move_TU1p s__3 = (const lzo_memops_move_TU1p) (const lzo_memops_TU0p) (ss); \
- d__3[0] = s__3[0]; d__3[1] = s__3[1]; d__3[2] = s__3[2]; \
- LZO_BLOCK_END
-#define LZO_MEMOPS_MOVE4(dd,ss) \
- LZO_BLOCK_BEGIN \
- lzo_memops_move_TU1p d__4 = (lzo_memops_move_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_move_TU1p s__4 = (const lzo_memops_move_TU1p) (const lzo_memops_TU0p) (ss); \
- d__4[0] = s__4[0]; d__4[1] = s__4[1]; d__4[2] = s__4[2]; d__4[3] = s__4[3]; \
- LZO_BLOCK_END
-#define LZO_MEMOPS_MOVE8(dd,ss) \
- LZO_BLOCK_BEGIN \
- lzo_memops_move_TU1p d__8 = (lzo_memops_move_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_move_TU1p s__8 = (const lzo_memops_move_TU1p) (const lzo_memops_TU0p) (ss); \
- d__8[0] = s__8[0]; d__8[1] = s__8[1]; d__8[2] = s__8[2]; d__8[3] = s__8[3]; \
- d__8[4] = s__8[4]; d__8[5] = s__8[5]; d__8[6] = s__8[6]; d__8[7] = s__8[7]; \
- LZO_BLOCK_END
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU1p)0)==1)
-#define LZO_MEMOPS_COPY1(dd,ss) LZO_MEMOPS_MOVE1(dd,ss)
-#if (LZO_OPT_UNALIGNED16)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU2p)0)==2)
-#define LZO_MEMOPS_COPY2(dd,ss) \
- * (lzo_memops_TU2p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU2p) (const lzo_memops_TU0p) (ss)
-#elif defined(lzo_memops_tcheck__)
-#define LZO_MEMOPS_COPY2(dd,ss) \
- LZO_BLOCK_BEGIN if (lzo_memops_tcheck__(lzo_memops_TU2,2,1)) { \
- * (lzo_memops_TU2p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU2p) (const lzo_memops_TU0p) (ss); \
- } else { LZO_MEMOPS_MOVE2(dd,ss); } LZO_BLOCK_END
-#else
-#define LZO_MEMOPS_COPY2(dd,ss) LZO_MEMOPS_MOVE2(dd,ss)
-#endif
-#if (LZO_OPT_UNALIGNED32)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU4p)0)==4)
-#define LZO_MEMOPS_COPY4(dd,ss) \
- * (lzo_memops_TU4p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU4p) (const lzo_memops_TU0p) (ss)
-#elif defined(lzo_memops_tcheck__)
-#define LZO_MEMOPS_COPY4(dd,ss) \
- LZO_BLOCK_BEGIN if (lzo_memops_tcheck__(lzo_memops_TU4,4,1)) { \
- * (lzo_memops_TU4p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU4p) (const lzo_memops_TU0p) (ss); \
- } else { LZO_MEMOPS_MOVE4(dd,ss); } LZO_BLOCK_END
-#else
-#define LZO_MEMOPS_COPY4(dd,ss) LZO_MEMOPS_MOVE4(dd,ss)
-#endif
-#if (LZO_WORDSIZE != 8)
-#define LZO_MEMOPS_COPY8(dd,ss) \
- LZO_BLOCK_BEGIN LZO_MEMOPS_COPY4(dd,ss); LZO_MEMOPS_COPY4((lzo_memops_TU1p)(lzo_memops_TU0p)(dd)+4,(const lzo_memops_TU1p)(const lzo_memops_TU0p)(ss)+4); LZO_BLOCK_END
-#else
-#if (LZO_OPT_UNALIGNED64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU8p)0)==8)
-#define LZO_MEMOPS_COPY8(dd,ss) \
- * (lzo_memops_TU8p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU8p) (const lzo_memops_TU0p) (ss)
-#elif (LZO_OPT_UNALIGNED32)
-#define LZO_MEMOPS_COPY8(dd,ss) \
- LZO_BLOCK_BEGIN LZO_MEMOPS_COPY4(dd,ss); LZO_MEMOPS_COPY4((lzo_memops_TU1p)(lzo_memops_TU0p)(dd)+4,(const lzo_memops_TU1p)(const lzo_memops_TU0p)(ss)+4); LZO_BLOCK_END
-#elif defined(lzo_memops_tcheck__)
-#define LZO_MEMOPS_COPY8(dd,ss) \
- LZO_BLOCK_BEGIN if (lzo_memops_tcheck__(lzo_memops_TU8,8,1)) { \
- * (lzo_memops_TU8p) (lzo_memops_TU0p) (dd) = * (const lzo_memops_TU8p) (const lzo_memops_TU0p) (ss); \
- } else { LZO_MEMOPS_MOVE8(dd,ss); } LZO_BLOCK_END
-#else
-#define LZO_MEMOPS_COPY8(dd,ss) LZO_MEMOPS_MOVE8(dd,ss)
-#endif
-#endif
-#define LZO_MEMOPS_COPYN(dd,ss,nn) \
- LZO_BLOCK_BEGIN \
- lzo_memops_TU1p d__n = (lzo_memops_TU1p) (lzo_memops_TU0p) (dd); \
- const lzo_memops_TU1p s__n = (const lzo_memops_TU1p) (const lzo_memops_TU0p) (ss); \
- lzo_uint n__n = (nn); \
- while ((void)0, n__n >= 8) { LZO_MEMOPS_COPY8(d__n, s__n); d__n += 8; s__n += 8; n__n -= 8; } \
- if ((void)0, n__n >= 4) { LZO_MEMOPS_COPY4(d__n, s__n); d__n += 4; s__n += 4; n__n -= 4; } \
- if ((void)0, n__n > 0) do { *d__n++ = *s__n++; } while (--n__n > 0); \
- LZO_BLOCK_END
-
-__lzo_static_forceinline lzo_uint16_t lzo_memops_get_le16(const lzo_voidp ss)
-{
- lzo_uint16_t v;
-#if (LZO_ABI_LITTLE_ENDIAN)
- LZO_MEMOPS_COPY2(&v, ss);
-#elif (LZO_OPT_UNALIGNED16 && LZO_ARCH_POWERPC && LZO_ABI_BIG_ENDIAN) && (LZO_ASM_SYNTAX_GNUC)
- const lzo_memops_TU2p s = (const lzo_memops_TU2p) ss;
- unsigned long vv;
- __asm__("lhbrx %0,0,%1" : "=r" (vv) : "r" (s), "m" (*s));
- v = (lzo_uint16_t) vv;
-#else
- const lzo_memops_TU1p s = (const lzo_memops_TU1p) ss;
- v = (lzo_uint16_t) (((lzo_uint16_t)s[0]) | ((lzo_uint16_t)s[1] << 8));
-#endif
- return v;
-}
-#if (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
-#define LZO_MEMOPS_GET_LE16(ss) (* (const lzo_memops_TU2p) (const lzo_memops_TU0p) (ss))
-#else
-#define LZO_MEMOPS_GET_LE16(ss) lzo_memops_get_le16(ss)
-#endif
-
-__lzo_static_forceinline lzo_uint32_t lzo_memops_get_le32(const lzo_voidp ss)
-{
- lzo_uint32_t v;
-#if (LZO_ABI_LITTLE_ENDIAN)
- LZO_MEMOPS_COPY4(&v, ss);
-#elif (LZO_OPT_UNALIGNED32 && LZO_ARCH_POWERPC && LZO_ABI_BIG_ENDIAN) && (LZO_ASM_SYNTAX_GNUC)
- const lzo_memops_TU4p s = (const lzo_memops_TU4p) ss;
- unsigned long vv;
- __asm__("lwbrx %0,0,%1" : "=r" (vv) : "r" (s), "m" (*s));
- v = (lzo_uint32_t) vv;
-#else
- const lzo_memops_TU1p s = (const lzo_memops_TU1p) ss;
- v = (lzo_uint32_t) (((lzo_uint32_t)s[0]) | ((lzo_uint32_t)s[1] << 8) | ((lzo_uint32_t)s[2] << 16) | ((lzo_uint32_t)s[3] << 24));
-#endif
- return v;
-}
-#if (LZO_OPT_UNALIGNED32) && (LZO_ABI_LITTLE_ENDIAN)
-#define LZO_MEMOPS_GET_LE32(ss) (* (const lzo_memops_TU4p) (const lzo_memops_TU0p) (ss))
-#else
-#define LZO_MEMOPS_GET_LE32(ss) lzo_memops_get_le32(ss)
-#endif
-
-#if (LZO_OPT_UNALIGNED64) && (LZO_ABI_LITTLE_ENDIAN)
-#define LZO_MEMOPS_GET_LE64(ss) (* (const lzo_memops_TU8p) (const lzo_memops_TU0p) (ss))
-#endif
-
-__lzo_static_forceinline lzo_uint16_t lzo_memops_get_ne16(const lzo_voidp ss)
-{
- lzo_uint16_t v;
- LZO_MEMOPS_COPY2(&v, ss);
- return v;
-}
-#if (LZO_OPT_UNALIGNED16)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU2p)0)==2)
-#define LZO_MEMOPS_GET_NE16(ss) (* (const lzo_memops_TU2p) (const lzo_memops_TU0p) (ss))
-#else
-#define LZO_MEMOPS_GET_NE16(ss) lzo_memops_get_ne16(ss)
-#endif
-
-__lzo_static_forceinline lzo_uint32_t lzo_memops_get_ne32(const lzo_voidp ss)
-{
- lzo_uint32_t v;
- LZO_MEMOPS_COPY4(&v, ss);
- return v;
-}
-#if (LZO_OPT_UNALIGNED32)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU4p)0)==4)
-#define LZO_MEMOPS_GET_NE32(ss) (* (const lzo_memops_TU4p) (const lzo_memops_TU0p) (ss))
-#else
-#define LZO_MEMOPS_GET_NE32(ss) lzo_memops_get_ne32(ss)
-#endif
-
-#if (LZO_OPT_UNALIGNED64)
-LZO_COMPILE_TIME_ASSERT_HEADER(sizeof(*(lzo_memops_TU8p)0)==8)
-#define LZO_MEMOPS_GET_NE64(ss) (* (const lzo_memops_TU8p) (const lzo_memops_TU0p) (ss))
-#endif
-
-__lzo_static_forceinline void lzo_memops_put_le16(lzo_voidp dd, lzo_uint16_t vv)
-{
-#if (LZO_ABI_LITTLE_ENDIAN)
- LZO_MEMOPS_COPY2(dd, &vv);
-#elif (LZO_OPT_UNALIGNED16 && LZO_ARCH_POWERPC && LZO_ABI_BIG_ENDIAN) && (LZO_ASM_SYNTAX_GNUC)
- lzo_memops_TU2p d = (lzo_memops_TU2p) dd;
- unsigned long v = vv;
- __asm__("sthbrx %2,0,%1" : "=m" (*d) : "r" (d), "r" (v));
-#else
- lzo_memops_TU1p d = (lzo_memops_TU1p) dd;
- d[0] = LZO_BYTE((vv ) & 0xff);
- d[1] = LZO_BYTE((vv >> 8) & 0xff);
-#endif
-}
-#if (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
-#define LZO_MEMOPS_PUT_LE16(dd,vv) (* (lzo_memops_TU2p) (lzo_memops_TU0p) (dd) = (vv))
-#else
-#define LZO_MEMOPS_PUT_LE16(dd,vv) lzo_memops_put_le16(dd,vv)
-#endif
-
-__lzo_static_forceinline void lzo_memops_put_le32(lzo_voidp dd, lzo_uint32_t vv)
-{
-#if (LZO_ABI_LITTLE_ENDIAN)
- LZO_MEMOPS_COPY4(dd, &vv);
-#elif (LZO_OPT_UNALIGNED32 && LZO_ARCH_POWERPC && LZO_ABI_BIG_ENDIAN) && (LZO_ASM_SYNTAX_GNUC)
- lzo_memops_TU4p d = (lzo_memops_TU4p) dd;
- unsigned long v = vv;
- __asm__("stwbrx %2,0,%1" : "=m" (*d) : "r" (d), "r" (v));
-#else
- lzo_memops_TU1p d = (lzo_memops_TU1p) dd;
- d[0] = LZO_BYTE((vv ) & 0xff);
- d[1] = LZO_BYTE((vv >> 8) & 0xff);
- d[2] = LZO_BYTE((vv >> 16) & 0xff);
- d[3] = LZO_BYTE((vv >> 24) & 0xff);
-#endif
-}
-#if (LZO_OPT_UNALIGNED32) && (LZO_ABI_LITTLE_ENDIAN)
-#define LZO_MEMOPS_PUT_LE32(dd,vv) (* (lzo_memops_TU4p) (lzo_memops_TU0p) (dd) = (vv))
-#else
-#define LZO_MEMOPS_PUT_LE32(dd,vv) lzo_memops_put_le32(dd,vv)
-#endif
-
-__lzo_static_forceinline void lzo_memops_put_ne16(lzo_voidp dd, lzo_uint16_t vv)
-{
- LZO_MEMOPS_COPY2(dd, &vv);
-}
-#if (LZO_OPT_UNALIGNED16)
-#define LZO_MEMOPS_PUT_NE16(dd,vv) (* (lzo_memops_TU2p) (lzo_memops_TU0p) (dd) = (vv))
-#else
-#define LZO_MEMOPS_PUT_NE16(dd,vv) lzo_memops_put_ne16(dd,vv)
-#endif
-
-__lzo_static_forceinline void lzo_memops_put_ne32(lzo_voidp dd, lzo_uint32_t vv)
-{
- LZO_MEMOPS_COPY4(dd, &vv);
-}
-#if (LZO_OPT_UNALIGNED32)
-#define LZO_MEMOPS_PUT_NE32(dd,vv) (* (lzo_memops_TU4p) (lzo_memops_TU0p) (dd) = (vv))
-#else
-#define LZO_MEMOPS_PUT_NE32(dd,vv) lzo_memops_put_ne32(dd,vv)
-#endif
-
-lzo_unused_funcs_impl(void, lzo_memops_unused_funcs)(void)
-{
- LZO_UNUSED_FUNC(lzo_memops_unused_funcs);
- LZO_UNUSED_FUNC(lzo_memops_get_le16);
- LZO_UNUSED_FUNC(lzo_memops_get_le32);
- LZO_UNUSED_FUNC(lzo_memops_get_ne16);
- LZO_UNUSED_FUNC(lzo_memops_get_ne32);
- LZO_UNUSED_FUNC(lzo_memops_put_le16);
- LZO_UNUSED_FUNC(lzo_memops_put_le32);
- LZO_UNUSED_FUNC(lzo_memops_put_ne16);
- LZO_UNUSED_FUNC(lzo_memops_put_ne32);
-}
-
-#endif
-
-#ifndef UA_SET1
-#define UA_SET1 LZO_MEMOPS_SET1
-#endif
-#ifndef UA_SET2
-#define UA_SET2 LZO_MEMOPS_SET2
-#endif
-#ifndef UA_SET3
-#define UA_SET3 LZO_MEMOPS_SET3
-#endif
-#ifndef UA_SET4
-#define UA_SET4 LZO_MEMOPS_SET4
-#endif
-#ifndef UA_MOVE1
-#define UA_MOVE1 LZO_MEMOPS_MOVE1
-#endif
-#ifndef UA_MOVE2
-#define UA_MOVE2 LZO_MEMOPS_MOVE2
-#endif
-#ifndef UA_MOVE3
-#define UA_MOVE3 LZO_MEMOPS_MOVE3
-#endif
-#ifndef UA_MOVE4
-#define UA_MOVE4 LZO_MEMOPS_MOVE4
-#endif
-#ifndef UA_MOVE8
-#define UA_MOVE8 LZO_MEMOPS_MOVE8
-#endif
-#ifndef UA_COPY1
-#define UA_COPY1 LZO_MEMOPS_COPY1
-#endif
-#ifndef UA_COPY2
-#define UA_COPY2 LZO_MEMOPS_COPY2
-#endif
-#ifndef UA_COPY3
-#define UA_COPY3 LZO_MEMOPS_COPY3
-#endif
-#ifndef UA_COPY4
-#define UA_COPY4 LZO_MEMOPS_COPY4
-#endif
-#ifndef UA_COPY8
-#define UA_COPY8 LZO_MEMOPS_COPY8
-#endif
-#ifndef UA_COPYN
-#define UA_COPYN LZO_MEMOPS_COPYN
-#endif
-#ifndef UA_COPYN_X
-#define UA_COPYN_X LZO_MEMOPS_COPYN
-#endif
-#ifndef UA_GET_LE16
-#define UA_GET_LE16 LZO_MEMOPS_GET_LE16
-#endif
-#ifndef UA_GET_LE32
-#define UA_GET_LE32 LZO_MEMOPS_GET_LE32
-#endif
-#ifdef LZO_MEMOPS_GET_LE64
-#ifndef UA_GET_LE64
-#define UA_GET_LE64 LZO_MEMOPS_GET_LE64
-#endif
-#endif
-#ifndef UA_GET_NE16
-#define UA_GET_NE16 LZO_MEMOPS_GET_NE16
-#endif
-#ifndef UA_GET_NE32
-#define UA_GET_NE32 LZO_MEMOPS_GET_NE32
-#endif
-#ifdef LZO_MEMOPS_GET_NE64
-#ifndef UA_GET_NE64
-#define UA_GET_NE64 LZO_MEMOPS_GET_NE64
-#endif
-#endif
-#ifndef UA_PUT_LE16
-#define UA_PUT_LE16 LZO_MEMOPS_PUT_LE16
-#endif
-#ifndef UA_PUT_LE32
-#define UA_PUT_LE32 LZO_MEMOPS_PUT_LE32
-#endif
-#ifndef UA_PUT_NE16
-#define UA_PUT_NE16 LZO_MEMOPS_PUT_NE16
-#endif
-#ifndef UA_PUT_NE32
-#define UA_PUT_NE32 LZO_MEMOPS_PUT_NE32
-#endif
-
-#define MEMCPY8_DS(dest,src,len) \
- lzo_memcpy(dest,src,len); dest += len; src += len
-
-#define BZERO8_PTR(s,l,n) \
- lzo_memset((lzo_voidp)(s),0,(lzo_uint)(l)*(n))
-
-#define MEMCPY_DS(dest,src,len) \
- do *dest++ = *src++; while (--len > 0)
-
-LZO_EXTERN(const lzo_bytep) lzo_copyright(void);
-
-#ifndef __LZO_PTR_H
-#define __LZO_PTR_H 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if (LZO_ARCH_I086)
-#error "LZO_ARCH_I086 is unsupported"
-#elif (LZO_MM_PVP)
-#error "LZO_MM_PVP is unsupported"
-#else
-#define PTR(a) ((lzo_uintptr_t) (a))
-#define PTR_LINEAR(a) PTR(a)
-#define PTR_ALIGNED_4(a) ((PTR_LINEAR(a) & 3) == 0)
-#define PTR_ALIGNED_8(a) ((PTR_LINEAR(a) & 7) == 0)
-#define PTR_ALIGNED2_4(a,b) (((PTR_LINEAR(a) | PTR_LINEAR(b)) & 3) == 0)
-#define PTR_ALIGNED2_8(a,b) (((PTR_LINEAR(a) | PTR_LINEAR(b)) & 7) == 0)
-#endif
-
-#define PTR_LT(a,b) (PTR(a) < PTR(b))
-#define PTR_GE(a,b) (PTR(a) >= PTR(b))
-#define PTR_DIFF(a,b) (PTR(a) - PTR(b))
-#define pd(a,b) ((lzo_uint) ((a)-(b)))
-
-LZO_EXTERN(lzo_uintptr_t)
-__lzo_ptr_linear(const lzo_voidp ptr);
-
-typedef union
-{
- char a_char;
- unsigned char a_uchar;
- short a_short;
- unsigned short a_ushort;
- int a_int;
- unsigned int a_uint;
- long a_long;
- unsigned long a_ulong;
- lzo_int a_lzo_int;
- lzo_uint a_lzo_uint;
- lzo_xint a_lzo_xint;
- lzo_int16_t a_lzo_int16_t;
- lzo_uint16_t a_lzo_uint16_t;
- lzo_int32_t a_lzo_int32_t;
- lzo_uint32_t a_lzo_uint32_t;
-#if defined(lzo_uint64_t)
- lzo_int64_t a_lzo_int64_t;
- lzo_uint64_t a_lzo_uint64_t;
-#endif
- size_t a_size_t;
- ptrdiff_t a_ptrdiff_t;
- lzo_uintptr_t a_lzo_uintptr_t;
- void * a_void_p;
- char * a_char_p;
- unsigned char * a_uchar_p;
- const void * a_c_void_p;
- const char * a_c_char_p;
- const unsigned char * a_c_uchar_p;
- lzo_voidp a_lzo_voidp;
- lzo_bytep a_lzo_bytep;
- const lzo_voidp a_c_lzo_voidp;
- const lzo_bytep a_c_lzo_bytep;
-}
-lzo_full_align_t;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-
-#ifndef LZO_DETERMINISTIC
-#define LZO_DETERMINISTIC 1
-#endif
-
-#ifndef LZO_DICT_USE_PTR
-#define LZO_DICT_USE_PTR 1
-#endif
-
-#if (LZO_DICT_USE_PTR)
-# define lzo_dict_t const lzo_bytep
-# define lzo_dict_p lzo_dict_t *
-#else
-# define lzo_dict_t lzo_uint
-# define lzo_dict_p lzo_dict_t *
-#endif
-
-#endif
-
-#if !defined(MINILZO_CFG_SKIP_LZO_PTR)
-
-LZO_PUBLIC(lzo_uintptr_t)
-__lzo_ptr_linear(const lzo_voidp ptr)
-{
- lzo_uintptr_t p;
-
-#if (LZO_ARCH_I086)
-#error "LZO_ARCH_I086 is unsupported"
-#elif (LZO_MM_PVP)
-#error "LZO_MM_PVP is unsupported"
-#else
- p = (lzo_uintptr_t) PTR_LINEAR(ptr);
-#endif
-
- return p;
-}
-
-LZO_PUBLIC(unsigned)
-__lzo_align_gap(const lzo_voidp ptr, lzo_uint size)
-{
-#if (__LZO_UINTPTR_T_IS_POINTER)
-#error "__LZO_UINTPTR_T_IS_POINTER is unsupported"
-#else
- lzo_uintptr_t p, n;
- if (size < 2) return 0;
- p = __lzo_ptr_linear(ptr);
-#if 0
- n = (((p + size - 1) / size) * size) - p;
-#else
- if ((size & (size - 1)) != 0)
- return 0;
- n = size; n = ((p + n - 1) & ~(n - 1)) - p;
-#endif
-#endif
- assert((long)n >= 0);
- assert(n <= size);
- return (unsigned)n;
-}
-
-#endif
-#if !defined(MINILZO_CFG_SKIP_LZO_UTIL)
-
-/* If you use the LZO library in a product, I would appreciate that you
- * keep this copyright string in the executable of your product.
- */
-
-static const char lzo_copyright_[] =
-#if !defined(__LZO_IN_MINLZO)
- LZO_VERSION_STRING;
-#else
- "\r\n\n"
- "LZO data compression library.\n"
- "$Copyright: LZO Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer\n"
- "<markus@oberhumer.com>\n"
- "http://www.oberhumer.com $\n\n"
- "$Id: LZO version: v" LZO_VERSION_STRING ", " LZO_VERSION_DATE " $\n"
- "$Info: " LZO_INFO_STRING " $\n";
-#endif
-static const char lzo_version_string_[] = LZO_VERSION_STRING;
-static const char lzo_version_date_[] = LZO_VERSION_DATE;
-
-LZO_PUBLIC(const lzo_bytep)
-lzo_copyright(void)
-{
- return (const lzo_bytep) lzo_copyright_;
-}
-
-LZO_PUBLIC(unsigned)
-lzo_version(void)
-{
- return LZO_VERSION;
-}
-
-LZO_PUBLIC(const char *)
-lzo_version_string(void)
-{
- return lzo_version_string_;
-}
-
-LZO_PUBLIC(const char *)
-lzo_version_date(void)
-{
- return lzo_version_date_;
-}
-
-LZO_PUBLIC(const lzo_charp)
-_lzo_version_string(void)
-{
- return lzo_version_string_;
-}
-
-LZO_PUBLIC(const lzo_charp)
-_lzo_version_date(void)
-{
- return lzo_version_date_;
-}
-
-#define LZO_BASE 65521u
-#define LZO_NMAX 5552
-
-#define LZO_DO1(buf,i) s1 += buf[i]; s2 += s1
-#define LZO_DO2(buf,i) LZO_DO1(buf,i); LZO_DO1(buf,i+1)
-#define LZO_DO4(buf,i) LZO_DO2(buf,i); LZO_DO2(buf,i+2)
-#define LZO_DO8(buf,i) LZO_DO4(buf,i); LZO_DO4(buf,i+4)
-#define LZO_DO16(buf,i) LZO_DO8(buf,i); LZO_DO8(buf,i+8)
-
-LZO_PUBLIC(lzo_uint32_t)
-lzo_adler32(lzo_uint32_t adler, const lzo_bytep buf, lzo_uint len)
-{
- lzo_uint32_t s1 = adler & 0xffff;
- lzo_uint32_t s2 = (adler >> 16) & 0xffff;
- unsigned k;
-
- if (buf == NULL)
- return 1;
-
- while (len > 0)
- {
- k = len < LZO_NMAX ? (unsigned) len : LZO_NMAX;
- len -= k;
- if (k >= 16) do
- {
- LZO_DO16(buf,0);
- buf += 16;
- k -= 16;
- } while (k >= 16);
- if (k != 0) do
- {
- s1 += *buf++;
- s2 += s1;
- } while (--k > 0);
- s1 %= LZO_BASE;
- s2 %= LZO_BASE;
- }
- return (s2 << 16) | s1;
-}
-
-#undef LZO_DO1
-#undef LZO_DO2
-#undef LZO_DO4
-#undef LZO_DO8
-#undef LZO_DO16
-
-#endif
-#if !defined(MINILZO_CFG_SKIP_LZO_STRING)
-#undef lzo_memcmp
-#undef lzo_memcpy
-#undef lzo_memmove
-#undef lzo_memset
-#if !defined(__LZO_MMODEL_HUGE)
-# undef LZO_HAVE_MM_HUGE_PTR
-#endif
-#define lzo_hsize_t lzo_uint
-#define lzo_hvoid_p lzo_voidp
-#define lzo_hbyte_p lzo_bytep
-#define LZOLIB_PUBLIC(r,f) LZO_PUBLIC(r) f
-#define lzo_hmemcmp lzo_memcmp
-#define lzo_hmemcpy lzo_memcpy
-#define lzo_hmemmove lzo_memmove
-#define lzo_hmemset lzo_memset
-#define __LZOLIB_HMEMCPY_CH_INCLUDED 1
-#if !defined(LZOLIB_PUBLIC)
-# define LZOLIB_PUBLIC(r,f) r __LZOLIB_FUNCNAME(f)
-#endif
-LZOLIB_PUBLIC(int, lzo_hmemcmp) (const lzo_hvoid_p s1, const lzo_hvoid_p s2, lzo_hsize_t len)
-{
-#if (LZO_HAVE_MM_HUGE_PTR) || !(HAVE_MEMCMP)
- const lzo_hbyte_p p1 = LZO_STATIC_CAST(const lzo_hbyte_p, s1);
- const lzo_hbyte_p p2 = LZO_STATIC_CAST(const lzo_hbyte_p, s2);
- if __lzo_likely(len > 0) do
- {
- int d = *p1 - *p2;
- if (d != 0)
- return d;
- p1++; p2++;
- } while __lzo_likely(--len > 0);
- return 0;
-#else
- return memcmp(s1, s2, len);
-#endif
-}
-LZOLIB_PUBLIC(lzo_hvoid_p, lzo_hmemcpy) (lzo_hvoid_p dest, const lzo_hvoid_p src, lzo_hsize_t len)
-{
-#if (LZO_HAVE_MM_HUGE_PTR) || !(HAVE_MEMCPY)
- lzo_hbyte_p p1 = LZO_STATIC_CAST(lzo_hbyte_p, dest);
- const lzo_hbyte_p p2 = LZO_STATIC_CAST(const lzo_hbyte_p, src);
- if (!(len > 0) || p1 == p2)
- return dest;
- do
- *p1++ = *p2++;
- while __lzo_likely(--len > 0);
- return dest;
-#else
- return memcpy(dest, src, len);
-#endif
-}
-LZOLIB_PUBLIC(lzo_hvoid_p, lzo_hmemmove) (lzo_hvoid_p dest, const lzo_hvoid_p src, lzo_hsize_t len)
-{
-#if (LZO_HAVE_MM_HUGE_PTR) || !(HAVE_MEMMOVE)
- lzo_hbyte_p p1 = LZO_STATIC_CAST(lzo_hbyte_p, dest);
- const lzo_hbyte_p p2 = LZO_STATIC_CAST(const lzo_hbyte_p, src);
- if (!(len > 0) || p1 == p2)
- return dest;
- if (p1 < p2)
- {
- do
- *p1++ = *p2++;
- while __lzo_likely(--len > 0);
- }
- else
- {
- p1 += len;
- p2 += len;
- do
- *--p1 = *--p2;
- while __lzo_likely(--len > 0);
- }
- return dest;
-#else
- return memmove(dest, src, len);
-#endif
-}
-LZOLIB_PUBLIC(lzo_hvoid_p, lzo_hmemset) (lzo_hvoid_p s, int cc, lzo_hsize_t len)
-{
-#if (LZO_HAVE_MM_HUGE_PTR) || !(HAVE_MEMSET)
- lzo_hbyte_p p = LZO_STATIC_CAST(lzo_hbyte_p, s);
- unsigned char c = LZO_ITRUNC(unsigned char, cc);
- if __lzo_likely(len > 0) do
- *p++ = c;
- while __lzo_likely(--len > 0);
- return s;
-#else
- return memset(s, cc, len);
-#endif
-}
-#undef LZOLIB_PUBLIC
-#endif
-#if !defined(MINILZO_CFG_SKIP_LZO_INIT)
-
-#if !defined(__LZO_IN_MINILZO)
-
-#define LZO_WANT_ACC_CHK_CH 1
-#undef LZOCHK_ASSERT
-
- LZOCHK_ASSERT((LZO_UINT32_C(1) << (int)(8*sizeof(LZO_UINT32_C(1))-1)) > 0)
- LZOCHK_ASSERT_IS_SIGNED_T(lzo_int)
- LZOCHK_ASSERT_IS_UNSIGNED_T(lzo_uint)
-#if !(__LZO_UINTPTR_T_IS_POINTER)
- LZOCHK_ASSERT_IS_UNSIGNED_T(lzo_uintptr_t)
-#endif
- LZOCHK_ASSERT(sizeof(lzo_uintptr_t) >= sizeof(lzo_voidp))
- LZOCHK_ASSERT_IS_UNSIGNED_T(lzo_xint)
-
-#endif
-#undef LZOCHK_ASSERT
-
-union lzo_config_check_union {
- lzo_uint a[2];
- unsigned char b[2*LZO_MAX(8,sizeof(lzo_uint))];
-#if defined(lzo_uint64_t)
- lzo_uint64_t c[2];
-#endif
-};
-
-#if 0
-#define u2p(ptr,off) ((lzo_voidp) (((lzo_bytep)(lzo_voidp)(ptr)) + (off)))
-#else
-static __lzo_noinline lzo_voidp u2p(lzo_voidp ptr, lzo_uint off)
-{
- return (lzo_voidp) ((lzo_bytep) ptr + off);
-}
-#endif
-
-LZO_PUBLIC(int)
-_lzo_config_check(void)
-{
-#if (LZO_CC_CLANG && (LZO_CC_CLANG >= 0x030100ul && LZO_CC_CLANG < 0x030300ul))
-# if 0
- volatile
-# endif
-#endif
- union lzo_config_check_union u;
- lzo_voidp p;
- unsigned r = 1;
-
- u.a[0] = u.a[1] = 0;
- p = u2p(&u, 0);
- r &= ((* (lzo_bytep) p) == 0);
-#if !(LZO_CFG_NO_CONFIG_CHECK)
-#if (LZO_ABI_BIG_ENDIAN)
- u.a[0] = u.a[1] = 0; u.b[sizeof(lzo_uint) - 1] = 128;
- p = u2p(&u, 0);
- r &= ((* (lzo_uintp) p) == 128);
-#endif
-#if (LZO_ABI_LITTLE_ENDIAN)
- u.a[0] = u.a[1] = 0; u.b[0] = 128;
- p = u2p(&u, 0);
- r &= ((* (lzo_uintp) p) == 128);
-#endif
- u.a[0] = u.a[1] = 0;
- u.b[0] = 1; u.b[3] = 2;
- p = u2p(&u, 1);
- r &= UA_GET_NE16(p) == 0;
- r &= UA_GET_LE16(p) == 0;
- u.b[1] = 128;
- r &= UA_GET_LE16(p) == 128;
- u.b[2] = 129;
- r &= UA_GET_LE16(p) == LZO_UINT16_C(0x8180);
-#if (LZO_ABI_BIG_ENDIAN)
- r &= UA_GET_NE16(p) == LZO_UINT16_C(0x8081);
-#endif
-#if (LZO_ABI_LITTLE_ENDIAN)
- r &= UA_GET_NE16(p) == LZO_UINT16_C(0x8180);
-#endif
- u.a[0] = u.a[1] = 0;
- u.b[0] = 3; u.b[5] = 4;
- p = u2p(&u, 1);
- r &= UA_GET_NE32(p) == 0;
- r &= UA_GET_LE32(p) == 0;
- u.b[1] = 128;
- r &= UA_GET_LE32(p) == 128;
- u.b[2] = 129; u.b[3] = 130; u.b[4] = 131;
- r &= UA_GET_LE32(p) == LZO_UINT32_C(0x83828180);
-#if (LZO_ABI_BIG_ENDIAN)
- r &= UA_GET_NE32(p) == LZO_UINT32_C(0x80818283);
-#endif
-#if (LZO_ABI_LITTLE_ENDIAN)
- r &= UA_GET_NE32(p) == LZO_UINT32_C(0x83828180);
-#endif
-#if defined(UA_GET_NE64)
- u.c[0] = u.c[1] = 0;
- u.b[0] = 5; u.b[9] = 6;
- p = u2p(&u, 1);
- u.c[0] = u.c[1] = 0;
- r &= UA_GET_NE64(p) == 0;
-#if defined(UA_GET_LE64)
- r &= UA_GET_LE64(p) == 0;
- u.b[1] = 128;
- r &= UA_GET_LE64(p) == 128;
-#endif
-#endif
-#if defined(lzo_bitops_ctlz32)
- { unsigned i = 0; lzo_uint32_t v;
- for (v = 1; v != 0 && r == 1; v <<= 1, i++) {
- r &= lzo_bitops_ctlz32(v) == 31 - i;
- r &= lzo_bitops_ctlz32_func(v) == 31 - i;
- }}
-#endif
-#if defined(lzo_bitops_ctlz64)
- { unsigned i = 0; lzo_uint64_t v;
- for (v = 1; v != 0 && r == 1; v <<= 1, i++) {
- r &= lzo_bitops_ctlz64(v) == 63 - i;
- r &= lzo_bitops_ctlz64_func(v) == 63 - i;
- }}
-#endif
-#if defined(lzo_bitops_cttz32)
- { unsigned i = 0; lzo_uint32_t v;
- for (v = 1; v != 0 && r == 1; v <<= 1, i++) {
- r &= lzo_bitops_cttz32(v) == i;
- r &= lzo_bitops_cttz32_func(v) == i;
- }}
-#endif
-#if defined(lzo_bitops_cttz64)
- { unsigned i = 0; lzo_uint64_t v;
- for (v = 1; v != 0 && r == 1; v <<= 1, i++) {
- r &= lzo_bitops_cttz64(v) == i;
- r &= lzo_bitops_cttz64_func(v) == i;
- }}
-#endif
-#endif
- LZO_UNUSED_FUNC(lzo_bitops_unused_funcs);
-
- return r == 1 ? LZO_E_OK : LZO_E_ERROR;
-}
-
-LZO_PUBLIC(int)
-__lzo_init_v2(unsigned v, int s1, int s2, int s3, int s4, int s5,
- int s6, int s7, int s8, int s9)
-{
- int r;
-
-#if defined(__LZO_IN_MINILZO)
-#elif (LZO_CC_MSC && ((_MSC_VER) < 700))
-#else
-#define LZO_WANT_ACC_CHK_CH 1
-#undef LZOCHK_ASSERT
-#define LZOCHK_ASSERT(expr) LZO_COMPILE_TIME_ASSERT(expr)
-#endif
-#undef LZOCHK_ASSERT
-
- if (v == 0)
- return LZO_E_ERROR;
-
- r = (s1 == -1 || s1 == (int) sizeof(short)) &&
- (s2 == -1 || s2 == (int) sizeof(int)) &&
- (s3 == -1 || s3 == (int) sizeof(long)) &&
- (s4 == -1 || s4 == (int) sizeof(lzo_uint32_t)) &&
- (s5 == -1 || s5 == (int) sizeof(lzo_uint)) &&
- (s6 == -1 || s6 == (int) lzo_sizeof_dict_t) &&
- (s7 == -1 || s7 == (int) sizeof(char *)) &&
- (s8 == -1 || s8 == (int) sizeof(lzo_voidp)) &&
- (s9 == -1 || s9 == (int) sizeof(lzo_callback_t));
- if (!r)
- return LZO_E_ERROR;
-
- r = _lzo_config_check();
- if (r != LZO_E_OK)
- return r;
-
- return r;
-}
-
-#if !defined(__LZO_IN_MINILZO)
-
-#if (LZO_OS_WIN16 && LZO_CC_WATCOMC) && defined(__SW_BD)
-
-#if 0
-BOOL FAR PASCAL LibMain ( HANDLE hInstance, WORD wDataSegment,
- WORD wHeapSize, LPSTR lpszCmdLine )
-#else
-int __far __pascal LibMain ( int a, short b, short c, long d )
-#endif
-{
- LZO_UNUSED(a); LZO_UNUSED(b); LZO_UNUSED(c); LZO_UNUSED(d);
- return 1;
-}
-
-#endif
-
-#endif
-
-#endif
-
-#define LZO1X 1
-#define LZO_EOF_CODE 1
-#define M2_MAX_OFFSET 0x0800
-
-#if !defined(MINILZO_CFG_SKIP_LZO1X_1_COMPRESS)
-
-#if 1 && defined(UA_GET_LE32)
-#undef LZO_DICT_USE_PTR
-#define LZO_DICT_USE_PTR 0
-#undef lzo_dict_t
-#define lzo_dict_t lzo_uint16_t
-#endif
-
-#define LZO_NEED_DICT_H 1
-#ifndef D_BITS
-#define D_BITS 14
-#endif
-#define D_INDEX1(d,p) d = DM(DMUL(0x21,DX3(p,5,5,6)) >> 5)
-#define D_INDEX2(d,p) d = (d & (D_MASK & 0x7ff)) ^ (D_HIGH | 0x1f)
-#if 1
-#define DINDEX(dv,p) DM(((DMUL(0x1824429d,dv)) >> (32-D_BITS)))
-#else
-#define DINDEX(dv,p) DM((dv) + ((dv) >> (32-D_BITS)))
-#endif
-
-#ifndef __LZO_CONFIG1X_H
-#define __LZO_CONFIG1X_H 1
-
-#if !defined(LZO1X) && !defined(LZO1Y) && !defined(LZO1Z)
-# define LZO1X 1
-#endif
-
-#if !defined(__LZO_IN_MINILZO)
-#include <lzo/lzo1x.h>
-#endif
-
-#ifndef LZO_EOF_CODE
-#define LZO_EOF_CODE 1
-#endif
-#undef LZO_DETERMINISTIC
-
-#define M1_MAX_OFFSET 0x0400
-#ifndef M2_MAX_OFFSET
-#define M2_MAX_OFFSET 0x0800
-#endif
-#define M3_MAX_OFFSET 0x4000
-#define M4_MAX_OFFSET 0xbfff
-
-#define MX_MAX_OFFSET (M1_MAX_OFFSET + M2_MAX_OFFSET)
-
-#define M1_MIN_LEN 2
-#define M1_MAX_LEN 2
-#define M2_MIN_LEN 3
-#ifndef M2_MAX_LEN
-#define M2_MAX_LEN 8
-#endif
-#define M3_MIN_LEN 3
-#define M3_MAX_LEN 33
-#define M4_MIN_LEN 3
-#define M4_MAX_LEN 9
-
-#define M1_MARKER 0
-#define M2_MARKER 64
-#define M3_MARKER 32
-#define M4_MARKER 16
-
-#ifndef MIN_LOOKAHEAD
-#define MIN_LOOKAHEAD (M2_MAX_LEN + 1)
-#endif
-
-#if defined(LZO_NEED_DICT_H)
-
-#ifndef LZO_HASH
-#define LZO_HASH LZO_HASH_LZO_INCREMENTAL_B
-#endif
-#define DL_MIN_LEN M2_MIN_LEN
-
-#ifndef __LZO_DICT_H
-#define __LZO_DICT_H 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if !defined(D_BITS) && defined(DBITS)
-# define D_BITS DBITS
-#endif
-#if !defined(D_BITS)
-# error "D_BITS is not defined"
-#endif
-#if (D_BITS < 16)
-# define D_SIZE LZO_SIZE(D_BITS)
-# define D_MASK LZO_MASK(D_BITS)
-#else
-# define D_SIZE LZO_USIZE(D_BITS)
-# define D_MASK LZO_UMASK(D_BITS)
-#endif
-#define D_HIGH ((D_MASK >> 1) + 1)
-
-#if !defined(DD_BITS)
-# define DD_BITS 0
-#endif
-#define DD_SIZE LZO_SIZE(DD_BITS)
-#define DD_MASK LZO_MASK(DD_BITS)
-
-#if !defined(DL_BITS)
-# define DL_BITS (D_BITS - DD_BITS)
-#endif
-#if (DL_BITS < 16)
-# define DL_SIZE LZO_SIZE(DL_BITS)
-# define DL_MASK LZO_MASK(DL_BITS)
-#else
-# define DL_SIZE LZO_USIZE(DL_BITS)
-# define DL_MASK LZO_UMASK(DL_BITS)
-#endif
-
-#if (D_BITS != DL_BITS + DD_BITS)
-# error "D_BITS does not match"
-#endif
-#if (D_BITS < 6 || D_BITS > 18)
-# error "invalid D_BITS"
-#endif
-#if (DL_BITS < 6 || DL_BITS > 20)
-# error "invalid DL_BITS"
-#endif
-#if (DD_BITS < 0 || DD_BITS > 6)
-# error "invalid DD_BITS"
-#endif
-
-#if !defined(DL_MIN_LEN)
-# define DL_MIN_LEN 3
-#endif
-#if !defined(DL_SHIFT)
-# define DL_SHIFT ((DL_BITS + (DL_MIN_LEN - 1)) / DL_MIN_LEN)
-#endif
-
-#define LZO_HASH_GZIP 1
-#define LZO_HASH_GZIP_INCREMENTAL 2
-#define LZO_HASH_LZO_INCREMENTAL_A 3
-#define LZO_HASH_LZO_INCREMENTAL_B 4
-
-#if !defined(LZO_HASH)
-# error "choose a hashing strategy"
-#endif
-
-#undef DM
-#undef DX
-
-#if (DL_MIN_LEN == 3)
-# define _DV2_A(p,shift1,shift2) \
- (((( (lzo_xint)((p)[0]) << shift1) ^ (p)[1]) << shift2) ^ (p)[2])
-# define _DV2_B(p,shift1,shift2) \
- (((( (lzo_xint)((p)[2]) << shift1) ^ (p)[1]) << shift2) ^ (p)[0])
-# define _DV3_B(p,shift1,shift2,shift3) \
- ((_DV2_B((p)+1,shift1,shift2) << (shift3)) ^ (p)[0])
-#elif (DL_MIN_LEN == 2)
-# define _DV2_A(p,shift1,shift2) \
- (( (lzo_xint)(p[0]) << shift1) ^ p[1])
-# define _DV2_B(p,shift1,shift2) \
- (( (lzo_xint)(p[1]) << shift1) ^ p[2])
-#else
-# error "invalid DL_MIN_LEN"
-#endif
-#define _DV_A(p,shift) _DV2_A(p,shift,shift)
-#define _DV_B(p,shift) _DV2_B(p,shift,shift)
-#define DA2(p,s1,s2) \
- (((((lzo_xint)((p)[2]) << (s2)) + (p)[1]) << (s1)) + (p)[0])
-#define DS2(p,s1,s2) \
- (((((lzo_xint)((p)[2]) << (s2)) - (p)[1]) << (s1)) - (p)[0])
-#define DX2(p,s1,s2) \
- (((((lzo_xint)((p)[2]) << (s2)) ^ (p)[1]) << (s1)) ^ (p)[0])
-#define DA3(p,s1,s2,s3) ((DA2((p)+1,s2,s3) << (s1)) + (p)[0])
-#define DS3(p,s1,s2,s3) ((DS2((p)+1,s2,s3) << (s1)) - (p)[0])
-#define DX3(p,s1,s2,s3) ((DX2((p)+1,s2,s3) << (s1)) ^ (p)[0])
-#define DMS(v,s) ((lzo_uint) (((v) & (D_MASK >> (s))) << (s)))
-#define DM(v) DMS(v,0)
-
-#if (LZO_HASH == LZO_HASH_GZIP)
-# define _DINDEX(dv,p) (_DV_A((p),DL_SHIFT))
-
-#elif (LZO_HASH == LZO_HASH_GZIP_INCREMENTAL)
-# define __LZO_HASH_INCREMENTAL 1
-# define DVAL_FIRST(dv,p) dv = _DV_A((p),DL_SHIFT)
-# define DVAL_NEXT(dv,p) dv = (((dv) << DL_SHIFT) ^ p[2])
-# define _DINDEX(dv,p) (dv)
-# define DVAL_LOOKAHEAD DL_MIN_LEN
-
-#elif (LZO_HASH == LZO_HASH_LZO_INCREMENTAL_A)
-# define __LZO_HASH_INCREMENTAL 1
-# define DVAL_FIRST(dv,p) dv = _DV_A((p),5)
-# define DVAL_NEXT(dv,p) \
- dv ^= (lzo_xint)(p[-1]) << (2*5); dv = (((dv) << 5) ^ p[2])
-# define _DINDEX(dv,p) ((DMUL(0x9f5f,dv)) >> 5)
-# define DVAL_LOOKAHEAD DL_MIN_LEN
-
-#elif (LZO_HASH == LZO_HASH_LZO_INCREMENTAL_B)
-# define __LZO_HASH_INCREMENTAL 1
-# define DVAL_FIRST(dv,p) dv = _DV_B((p),5)
-# define DVAL_NEXT(dv,p) \
- dv ^= p[-1]; dv = (((dv) >> 5) ^ ((lzo_xint)(p[2]) << (2*5)))
-# define _DINDEX(dv,p) ((DMUL(0x9f5f,dv)) >> 5)
-# define DVAL_LOOKAHEAD DL_MIN_LEN
-
-#else
-# error "choose a hashing strategy"
-#endif
-
-#ifndef DINDEX
-#define DINDEX(dv,p) ((lzo_uint)((_DINDEX(dv,p)) & DL_MASK) << DD_BITS)
-#endif
-#if !defined(DINDEX1) && defined(D_INDEX1)
-#define DINDEX1 D_INDEX1
-#endif
-#if !defined(DINDEX2) && defined(D_INDEX2)
-#define DINDEX2 D_INDEX2
-#endif
-
-#if !defined(__LZO_HASH_INCREMENTAL)
-# define DVAL_FIRST(dv,p) ((void) 0)
-# define DVAL_NEXT(dv,p) ((void) 0)
-# define DVAL_LOOKAHEAD 0
-#endif
-
-#if !defined(DVAL_ASSERT)
-#if defined(__LZO_HASH_INCREMENTAL) && !defined(NDEBUG)
-#if 1 && (LZO_CC_ARMCC_GNUC || LZO_CC_CLANG || (LZO_CC_GNUC >= 0x020700ul) || LZO_CC_INTELC_GNUC || LZO_CC_LLVM || LZO_CC_PATHSCALE || LZO_CC_PGI)
-static void __attribute__((__unused__))
-#else
-static void
-#endif
-DVAL_ASSERT(lzo_xint dv, const lzo_bytep p)
-{
- lzo_xint df;
- DVAL_FIRST(df,(p));
- assert(DINDEX(dv,p) == DINDEX(df,p));
-}
-#else
-# define DVAL_ASSERT(dv,p) ((void) 0)
-#endif
-#endif
-
-#if (LZO_DICT_USE_PTR)
-# define DENTRY(p,in) (p)
-# define GINDEX(m_pos,m_off,dict,dindex,in) m_pos = dict[dindex]
-#else
-# define DENTRY(p,in) ((lzo_dict_t) pd(p, in))
-# define GINDEX(m_pos,m_off,dict,dindex,in) m_off = dict[dindex]
-#endif
-
-#if (DD_BITS == 0)
-
-# define UPDATE_D(dict,drun,dv,p,in) dict[ DINDEX(dv,p) ] = DENTRY(p,in)
-# define UPDATE_I(dict,drun,index,p,in) dict[index] = DENTRY(p,in)
-# define UPDATE_P(ptr,drun,p,in) (ptr)[0] = DENTRY(p,in)
-
-#else
-
-# define UPDATE_D(dict,drun,dv,p,in) \
- dict[ DINDEX(dv,p) + drun++ ] = DENTRY(p,in); drun &= DD_MASK
-# define UPDATE_I(dict,drun,index,p,in) \
- dict[ (index) + drun++ ] = DENTRY(p,in); drun &= DD_MASK
-# define UPDATE_P(ptr,drun,p,in) \
- (ptr) [ drun++ ] = DENTRY(p,in); drun &= DD_MASK
-
-#endif
-
-#if (LZO_DICT_USE_PTR)
-
-#define LZO_CHECK_MPOS_DET(m_pos,m_off,in,ip,max_offset) \
- (m_pos == NULL || (m_off = pd(ip, m_pos)) > max_offset)
-
-#define LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,max_offset) \
- (BOUNDS_CHECKING_OFF_IN_EXPR(( \
- m_pos = ip - (lzo_uint) PTR_DIFF(ip,m_pos), \
- PTR_LT(m_pos,in) || \
- (m_off = (lzo_uint) PTR_DIFF(ip,m_pos)) == 0 || \
- m_off > max_offset )))
-
-#else
-
-#define LZO_CHECK_MPOS_DET(m_pos,m_off,in,ip,max_offset) \
- (m_off == 0 || \
- ((m_off = pd(ip, in) - m_off) > max_offset) || \
- (m_pos = (ip) - (m_off), 0) )
-
-#define LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,max_offset) \
- (pd(ip, in) <= m_off || \
- ((m_off = pd(ip, in) - m_off) > max_offset) || \
- (m_pos = (ip) - (m_off), 0) )
-
-#endif
-
-#if (LZO_DETERMINISTIC)
-# define LZO_CHECK_MPOS LZO_CHECK_MPOS_DET
-#else
-# define LZO_CHECK_MPOS LZO_CHECK_MPOS_NON_DET
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-
-#endif
-
-#endif
-
-#define LZO_DETERMINISTIC !(LZO_DICT_USE_PTR)
-
-#ifndef DO_COMPRESS
-#define DO_COMPRESS lzo1x_1_compress
-#endif
-
-#if 1 && defined(DO_COMPRESS) && !defined(do_compress)
-# define do_compress LZO_PP_ECONCAT2(DO_COMPRESS,_core)
-#endif
-
-static __lzo_noinline lzo_uint
-do_compress ( const lzo_bytep in , lzo_uint in_len,
- lzo_bytep out, lzo_uintp out_len,
- lzo_uint ti, lzo_voidp wrkmem)
-{
- const lzo_bytep ip;
- lzo_bytep op;
- const lzo_bytep const in_end = in + in_len;
- const lzo_bytep const ip_end = in + in_len - 20;
- const lzo_bytep ii;
- lzo_dict_p const dict = (lzo_dict_p) wrkmem;
-
- op = out;
- ip = in;
- ii = ip;
-
- ip += ti < 4 ? 4 - ti : 0;
- for (;;)
- {
- const lzo_bytep m_pos;
-#if !(LZO_DETERMINISTIC)
- LZO_DEFINE_UNINITIALIZED_VAR(lzo_uint, m_off, 0);
- lzo_uint m_len;
- lzo_uint dindex;
-next:
- if __lzo_unlikely(ip >= ip_end)
- break;
- DINDEX1(dindex,ip);
- GINDEX(m_pos,m_off,dict,dindex,in);
- if (LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,M4_MAX_OFFSET))
- goto literal;
-#if 1
- if (m_off <= M2_MAX_OFFSET || m_pos[3] == ip[3])
- goto try_match;
- DINDEX2(dindex,ip);
-#endif
- GINDEX(m_pos,m_off,dict,dindex,in);
- if (LZO_CHECK_MPOS_NON_DET(m_pos,m_off,in,ip,M4_MAX_OFFSET))
- goto literal;
- if (m_off <= M2_MAX_OFFSET || m_pos[3] == ip[3])
- goto try_match;
- goto literal;
-
-try_match:
-#if (LZO_OPT_UNALIGNED32)
- if (UA_GET_NE32(m_pos) != UA_GET_NE32(ip))
-#else
- if (m_pos[0] != ip[0] || m_pos[1] != ip[1] || m_pos[2] != ip[2] || m_pos[3] != ip[3])
-#endif
- {
-literal:
- UPDATE_I(dict,0,dindex,ip,in);
- ip += 1 + ((ip - ii) >> 5);
- continue;
- }
- UPDATE_I(dict,0,dindex,ip,in);
-#else
- lzo_uint m_off;
- lzo_uint m_len;
- {
- lzo_uint32_t dv;
- lzo_uint dindex;
-literal:
- ip += 1 + ((ip - ii) >> 5);
-next:
- if __lzo_unlikely(ip >= ip_end)
- break;
- dv = UA_GET_LE32(ip);
- dindex = DINDEX(dv,ip);
- GINDEX(m_off,m_pos,in+dict,dindex,in);
- UPDATE_I(dict,0,dindex,ip,in);
- if __lzo_unlikely(dv != UA_GET_LE32(m_pos))
- goto literal;
- }
-#endif
-
- ii -= ti; ti = 0;
- {
- lzo_uint t = pd(ip,ii);
- if (t != 0)
- {
- if (t <= 3)
- {
- op[-2] = LZO_BYTE(op[-2] | t);
-#if (LZO_OPT_UNALIGNED32)
- UA_COPY4(op, ii);
- op += t;
-#else
- { do *op++ = *ii++; while (--t > 0); }
-#endif
- }
-#if (LZO_OPT_UNALIGNED32) || (LZO_OPT_UNALIGNED64)
- else if (t <= 16)
- {
- *op++ = LZO_BYTE(t - 3);
- UA_COPY8(op, ii);
- UA_COPY8(op+8, ii+8);
- op += t;
- }
-#endif
- else
- {
- if (t <= 18)
- *op++ = LZO_BYTE(t - 3);
- else
- {
- lzo_uint tt = t - 18;
- *op++ = 0;
- while __lzo_unlikely(tt > 255)
- {
- tt -= 255;
- UA_SET1(op, 0);
- op++;
- }
- assert(tt > 0);
- *op++ = LZO_BYTE(tt);
- }
-#if (LZO_OPT_UNALIGNED32) || (LZO_OPT_UNALIGNED64)
- do {
- UA_COPY8(op, ii);
- UA_COPY8(op+8, ii+8);
- op += 16; ii += 16; t -= 16;
- } while (t >= 16); if (t > 0)
-#endif
- { do *op++ = *ii++; while (--t > 0); }
- }
- }
- }
- m_len = 4;
- {
-#if (LZO_OPT_UNALIGNED64)
- lzo_uint64_t v;
- v = UA_GET_NE64(ip + m_len) ^ UA_GET_NE64(m_pos + m_len);
- if __lzo_unlikely(v == 0) {
- do {
- m_len += 8;
- v = UA_GET_NE64(ip + m_len) ^ UA_GET_NE64(m_pos + m_len);
- if __lzo_unlikely(ip + m_len >= ip_end)
- goto m_len_done;
- } while (v == 0);
- }
-#if (LZO_ABI_BIG_ENDIAN) && defined(lzo_bitops_ctlz64)
- m_len += lzo_bitops_ctlz64(v) / CHAR_BIT;
-#elif (LZO_ABI_BIG_ENDIAN)
- if ((v >> (64 - CHAR_BIT)) == 0) do {
- v <<= CHAR_BIT;
- m_len += 1;
- } while ((v >> (64 - CHAR_BIT)) == 0);
-#elif (LZO_ABI_LITTLE_ENDIAN) && defined(lzo_bitops_cttz64)
- m_len += lzo_bitops_cttz64(v) / CHAR_BIT;
-#elif (LZO_ABI_LITTLE_ENDIAN)
- if ((v & UCHAR_MAX) == 0) do {
- v >>= CHAR_BIT;
- m_len += 1;
- } while ((v & UCHAR_MAX) == 0);
-#else
- if (ip[m_len] == m_pos[m_len]) do {
- m_len += 1;
- } while (ip[m_len] == m_pos[m_len]);
-#endif
-#elif (LZO_OPT_UNALIGNED32)
- lzo_uint32_t v;
- v = UA_GET_NE32(ip + m_len) ^ UA_GET_NE32(m_pos + m_len);
- if __lzo_unlikely(v == 0) {
- do {
- m_len += 4;
- v = UA_GET_NE32(ip + m_len) ^ UA_GET_NE32(m_pos + m_len);
- if (v != 0)
- break;
- m_len += 4;
- v = UA_GET_NE32(ip + m_len) ^ UA_GET_NE32(m_pos + m_len);
- if __lzo_unlikely(ip + m_len >= ip_end)
- goto m_len_done;
- } while (v == 0);
- }
-#if (LZO_ABI_BIG_ENDIAN) && defined(lzo_bitops_ctlz32)
- m_len += lzo_bitops_ctlz32(v) / CHAR_BIT;
-#elif (LZO_ABI_BIG_ENDIAN)
- if ((v >> (32 - CHAR_BIT)) == 0) do {
- v <<= CHAR_BIT;
- m_len += 1;
- } while ((v >> (32 - CHAR_BIT)) == 0);
-#elif (LZO_ABI_LITTLE_ENDIAN) && defined(lzo_bitops_cttz32)
- m_len += lzo_bitops_cttz32(v) / CHAR_BIT;
-#elif (LZO_ABI_LITTLE_ENDIAN)
- if ((v & UCHAR_MAX) == 0) do {
- v >>= CHAR_BIT;
- m_len += 1;
- } while ((v & UCHAR_MAX) == 0);
-#else
- if (ip[m_len] == m_pos[m_len]) do {
- m_len += 1;
- } while (ip[m_len] == m_pos[m_len]);
-#endif
-#else
- if __lzo_unlikely(ip[m_len] == m_pos[m_len]) {
- do {
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if (ip[m_len] != m_pos[m_len])
- break;
- m_len += 1;
- if __lzo_unlikely(ip + m_len >= ip_end)
- goto m_len_done;
- } while (ip[m_len] == m_pos[m_len]);
- }
-#endif
- }
-m_len_done:
- m_off = pd(ip,m_pos);
- ip += m_len;
- ii = ip;
- if (m_len <= M2_MAX_LEN && m_off <= M2_MAX_OFFSET)
- {
- m_off -= 1;
-#if defined(LZO1X)
- *op++ = LZO_BYTE(((m_len - 1) << 5) | ((m_off & 7) << 2));
- *op++ = LZO_BYTE(m_off >> 3);
-#elif defined(LZO1Y)
- *op++ = LZO_BYTE(((m_len + 1) << 4) | ((m_off & 3) << 2));
- *op++ = LZO_BYTE(m_off >> 2);
-#endif
- }
- else if (m_off <= M3_MAX_OFFSET)
- {
- m_off -= 1;
- if (m_len <= M3_MAX_LEN)
- *op++ = LZO_BYTE(M3_MARKER | (m_len - 2));
- else
- {
- m_len -= M3_MAX_LEN;
- *op++ = M3_MARKER | 0;
- while __lzo_unlikely(m_len > 255)
- {
- m_len -= 255;
- UA_SET1(op, 0);
- op++;
- }
- *op++ = LZO_BYTE(m_len);
- }
- *op++ = LZO_BYTE(m_off << 2);
- *op++ = LZO_BYTE(m_off >> 6);
- }
- else
- {
- m_off -= 0x4000;
- if (m_len <= M4_MAX_LEN)
- *op++ = LZO_BYTE(M4_MARKER | ((m_off >> 11) & 8) | (m_len - 2));
- else
- {
- m_len -= M4_MAX_LEN;
- *op++ = LZO_BYTE(M4_MARKER | ((m_off >> 11) & 8));
- while __lzo_unlikely(m_len > 255)
- {
- m_len -= 255;
- UA_SET1(op, 0);
- op++;
- }
- *op++ = LZO_BYTE(m_len);
- }
- *op++ = LZO_BYTE(m_off << 2);
- *op++ = LZO_BYTE(m_off >> 6);
- }
- goto next;
- }
-
- *out_len = pd(op, out);
- return pd(in_end,ii-ti);
-}
-
-LZO_PUBLIC(int)
-DO_COMPRESS ( const lzo_bytep in , lzo_uint in_len,
- lzo_bytep out, lzo_uintp out_len,
- lzo_voidp wrkmem )
-{
- const lzo_bytep ip = in;
- lzo_bytep op = out;
- lzo_uint l = in_len;
- lzo_uint t = 0;
-
- while (l > 20)
- {
- lzo_uint ll = l;
- lzo_uintptr_t ll_end;
-#if 0 || (LZO_DETERMINISTIC)
- ll = LZO_MIN(ll, 49152);
-#endif
- ll_end = (lzo_uintptr_t)ip + ll;
- if ((ll_end + ((t + ll) >> 5)) <= ll_end || (const lzo_bytep)(ll_end + ((t + ll) >> 5)) <= ip + ll)
- break;
-#if (LZO_DETERMINISTIC)
- lzo_memset(wrkmem, 0, ((lzo_uint)1 << D_BITS) * sizeof(lzo_dict_t));
-#endif
- t = do_compress(ip,ll,op,out_len,t,wrkmem);
- ip += ll;
- op += *out_len;
- l -= ll;
- }
- t += l;
-
- if (t > 0)
- {
- const lzo_bytep ii = in + in_len - t;
-
- if (op == out && t <= 238)
- *op++ = LZO_BYTE(17 + t);
- else if (t <= 3)
- op[-2] = LZO_BYTE(op[-2] | t);
- else if (t <= 18)
- *op++ = LZO_BYTE(t - 3);
- else
- {
- lzo_uint tt = t - 18;
-
- *op++ = 0;
- while (tt > 255)
- {
- tt -= 255;
- UA_SET1(op, 0);
- op++;
- }
- assert(tt > 0);
- *op++ = LZO_BYTE(tt);
- }
- UA_COPYN(op, ii, t);
- op += t;
- }
-
- *op++ = M4_MARKER | 1;
- *op++ = 0;
- *op++ = 0;
-
- *out_len = pd(op, out);
- return LZO_E_OK;
-}
-
-#endif
-
-#undef do_compress
-#undef DO_COMPRESS
-#undef LZO_HASH
-
-#undef LZO_TEST_OVERRUN
-#undef DO_DECOMPRESS
-#define DO_DECOMPRESS lzo1x_decompress
-
-#if !defined(MINILZO_CFG_SKIP_LZO1X_DECOMPRESS)
-
-#if defined(LZO_TEST_OVERRUN)
-# if !defined(LZO_TEST_OVERRUN_INPUT)
-# define LZO_TEST_OVERRUN_INPUT 2
-# endif
-# if !defined(LZO_TEST_OVERRUN_OUTPUT)
-# define LZO_TEST_OVERRUN_OUTPUT 2
-# endif
-# if !defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-# define LZO_TEST_OVERRUN_LOOKBEHIND 1
-# endif
-#endif
-
-#undef TEST_IP
-#undef TEST_OP
-#undef TEST_IP_AND_TEST_OP
-#undef TEST_LB
-#undef TEST_LBO
-#undef NEED_IP
-#undef NEED_OP
-#undef TEST_IV
-#undef TEST_OV
-#undef HAVE_TEST_IP
-#undef HAVE_TEST_OP
-#undef HAVE_NEED_IP
-#undef HAVE_NEED_OP
-#undef HAVE_ANY_IP
-#undef HAVE_ANY_OP
-
-#if defined(LZO_TEST_OVERRUN_INPUT)
-# if (LZO_TEST_OVERRUN_INPUT >= 1)
-# define TEST_IP (ip < ip_end)
-# endif
-# if (LZO_TEST_OVERRUN_INPUT >= 2)
-# define NEED_IP(x) \
- if ((lzo_uint)(ip_end - ip) < (lzo_uint)(x)) goto input_overrun
-# define TEST_IV(x) if ((x) > (lzo_uint)0 - (511)) goto input_overrun
-# endif
-#endif
-
-#if defined(LZO_TEST_OVERRUN_OUTPUT)
-# if (LZO_TEST_OVERRUN_OUTPUT >= 1)
-# define TEST_OP (op <= op_end)
-# endif
-# if (LZO_TEST_OVERRUN_OUTPUT >= 2)
-# undef TEST_OP
-# define NEED_OP(x) \
- if ((lzo_uint)(op_end - op) < (lzo_uint)(x)) goto output_overrun
-# define TEST_OV(x) if ((x) > (lzo_uint)0 - (511)) goto output_overrun
-# endif
-#endif
-
-#if defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-# define TEST_LB(m_pos) if (PTR_LT(m_pos,out) || PTR_GE(m_pos,op)) goto lookbehind_overrun
-# define TEST_LBO(m_pos,o) if (PTR_LT(m_pos,out) || PTR_GE(m_pos,op-(o))) goto lookbehind_overrun
-#else
-# define TEST_LB(m_pos) ((void) 0)
-# define TEST_LBO(m_pos,o) ((void) 0)
-#endif
-
-#if !defined(LZO_EOF_CODE) && !defined(TEST_IP)
-# define TEST_IP (ip < ip_end)
-#endif
-
-#if defined(TEST_IP)
-# define HAVE_TEST_IP 1
-#else
-# define TEST_IP 1
-#endif
-#if defined(TEST_OP)
-# define HAVE_TEST_OP 1
-#else
-# define TEST_OP 1
-#endif
-
-#if defined(HAVE_TEST_IP) && defined(HAVE_TEST_OP)
-# define TEST_IP_AND_TEST_OP (TEST_IP && TEST_OP)
-#elif defined(HAVE_TEST_IP)
-# define TEST_IP_AND_TEST_OP TEST_IP
-#elif defined(HAVE_TEST_OP)
-# define TEST_IP_AND_TEST_OP TEST_OP
-#else
-# define TEST_IP_AND_TEST_OP 1
-#endif
-
-#if defined(NEED_IP)
-# define HAVE_NEED_IP 1
-#else
-# define NEED_IP(x) ((void) 0)
-# define TEST_IV(x) ((void) 0)
-#endif
-#if defined(NEED_OP)
-# define HAVE_NEED_OP 1
-#else
-# define NEED_OP(x) ((void) 0)
-# define TEST_OV(x) ((void) 0)
-#endif
-
-#if defined(HAVE_TEST_IP) || defined(HAVE_NEED_IP)
-# define HAVE_ANY_IP 1
-#endif
-#if defined(HAVE_TEST_OP) || defined(HAVE_NEED_OP)
-# define HAVE_ANY_OP 1
-#endif
-
-#if defined(DO_DECOMPRESS)
-LZO_PUBLIC(int)
-DO_DECOMPRESS ( const lzo_bytep in , lzo_uint in_len,
- lzo_bytep out, lzo_uintp out_len,
- lzo_voidp wrkmem )
-#endif
-{
- lzo_bytep op;
- const lzo_bytep ip;
- lzo_uint t;
-#if defined(COPY_DICT)
- lzo_uint m_off;
- const lzo_bytep dict_end;
-#else
- const lzo_bytep m_pos;
-#endif
-
- const lzo_bytep const ip_end = in + in_len;
-#if defined(HAVE_ANY_OP)
- lzo_bytep const op_end = out + *out_len;
-#endif
-#if defined(LZO1Z)
- lzo_uint last_m_off = 0;
-#endif
-
- LZO_UNUSED(wrkmem);
-
-#if defined(COPY_DICT)
- if (dict)
- {
- if (dict_len > M4_MAX_OFFSET)
- {
- dict += dict_len - M4_MAX_OFFSET;
- dict_len = M4_MAX_OFFSET;
- }
- dict_end = dict + dict_len;
- }
- else
- {
- dict_len = 0;
- dict_end = NULL;
- }
-#endif
-
- *out_len = 0;
-
- op = out;
- ip = in;
-
- NEED_IP(1);
- if (*ip > 17)
- {
- t = *ip++ - 17;
- if (t < 4)
- goto match_next;
- assert(t > 0); NEED_OP(t); NEED_IP(t+3);
- do *op++ = *ip++; while (--t > 0);
- goto first_literal_run;
- }
-
- for (;;)
- {
- NEED_IP(3);
- t = *ip++;
- if (t >= 16)
- goto match;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_IV(t);
- NEED_IP(1);
- }
- t += 15 + *ip++;
- }
- assert(t > 0); NEED_OP(t+3); NEED_IP(t+6);
-#if (LZO_OPT_UNALIGNED64) && (LZO_OPT_UNALIGNED32)
- t += 3;
- if (t >= 8) do
- {
- UA_COPY8(op,ip);
- op += 8; ip += 8; t -= 8;
- } while (t >= 8);
- if (t >= 4)
- {
- UA_COPY4(op,ip);
- op += 4; ip += 4; t -= 4;
- }
- if (t > 0)
- {
- *op++ = *ip++;
- if (t > 1) { *op++ = *ip++; if (t > 2) { *op++ = *ip++; } }
- }
-#elif (LZO_OPT_UNALIGNED32) || (LZO_ALIGNED_OK_4)
-#if !(LZO_OPT_UNALIGNED32)
- if (PTR_ALIGNED2_4(op,ip))
- {
-#endif
- UA_COPY4(op,ip);
- op += 4; ip += 4;
- if (--t > 0)
- {
- if (t >= 4)
- {
- do {
- UA_COPY4(op,ip);
- op += 4; ip += 4; t -= 4;
- } while (t >= 4);
- if (t > 0) do *op++ = *ip++; while (--t > 0);
- }
- else
- do *op++ = *ip++; while (--t > 0);
- }
-#if !(LZO_OPT_UNALIGNED32)
- }
- else
-#endif
-#endif
-#if !(LZO_OPT_UNALIGNED32)
- {
- *op++ = *ip++; *op++ = *ip++; *op++ = *ip++;
- do *op++ = *ip++; while (--t > 0);
- }
-#endif
-
-first_literal_run:
-
- t = *ip++;
- if (t >= 16)
- goto match;
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2);
- last_m_off = m_off;
-#else
- m_off = (1 + M2_MAX_OFFSET) + (t >> 2) + (*ip++ << 2);
-#endif
- NEED_OP(3);
- t = 3; COPY_DICT(t,m_off)
-#else
-#if defined(LZO1Z)
- t = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2);
- m_pos = op - t;
- last_m_off = t;
-#else
- m_pos = op - (1 + M2_MAX_OFFSET);
- m_pos -= t >> 2;
- m_pos -= *ip++ << 2;
-#endif
- TEST_LB(m_pos); NEED_OP(3);
- *op++ = *m_pos++; *op++ = *m_pos++; *op++ = *m_pos;
-#endif
- goto match_done;
-
- for (;;) {
-match:
- if (t >= 64)
- {
-#if defined(COPY_DICT)
-#if defined(LZO1X)
- m_off = 1 + ((t >> 2) & 7) + (*ip++ << 3);
- t = (t >> 5) - 1;
-#elif defined(LZO1Y)
- m_off = 1 + ((t >> 2) & 3) + (*ip++ << 2);
- t = (t >> 4) - 3;
-#elif defined(LZO1Z)
- m_off = t & 0x1f;
- if (m_off >= 0x1c)
- m_off = last_m_off;
- else
- {
- m_off = 1 + (m_off << 6) + (*ip++ >> 2);
- last_m_off = m_off;
- }
- t = (t >> 5) - 1;
-#endif
-#else
-#if defined(LZO1X)
- m_pos = op - 1;
- m_pos -= (t >> 2) & 7;
- m_pos -= *ip++ << 3;
- t = (t >> 5) - 1;
-#elif defined(LZO1Y)
- m_pos = op - 1;
- m_pos -= (t >> 2) & 3;
- m_pos -= *ip++ << 2;
- t = (t >> 4) - 3;
-#elif defined(LZO1Z)
- {
- lzo_uint off = t & 0x1f;
- m_pos = op;
- if (off >= 0x1c)
- {
- assert(last_m_off > 0);
- m_pos -= last_m_off;
- }
- else
- {
- off = 1 + (off << 6) + (*ip++ >> 2);
- m_pos -= off;
- last_m_off = off;
- }
- }
- t = (t >> 5) - 1;
-#endif
- TEST_LB(m_pos); assert(t > 0); NEED_OP(t+3-1);
- goto copy_match;
-#endif
- }
- else if (t >= 32)
- {
- t &= 31;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_OV(t);
- NEED_IP(1);
- }
- t += 31 + *ip++;
- NEED_IP(2);
- }
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = 1 + (ip[0] << 6) + (ip[1] >> 2);
- last_m_off = m_off;
-#else
- m_off = 1 + (ip[0] >> 2) + (ip[1] << 6);
-#endif
-#else
-#if defined(LZO1Z)
- {
- lzo_uint off = 1 + (ip[0] << 6) + (ip[1] >> 2);
- m_pos = op - off;
- last_m_off = off;
- }
-#elif (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
- m_pos = op - 1;
- m_pos -= UA_GET_LE16(ip) >> 2;
-#else
- m_pos = op - 1;
- m_pos -= (ip[0] >> 2) + (ip[1] << 6);
-#endif
-#endif
- ip += 2;
- }
- else if (t >= 16)
- {
-#if defined(COPY_DICT)
- m_off = (t & 8) << 11;
-#else
- m_pos = op;
- m_pos -= (t & 8) << 11;
-#endif
- t &= 7;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_OV(t);
- NEED_IP(1);
- }
- t += 7 + *ip++;
- NEED_IP(2);
- }
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off += (ip[0] << 6) + (ip[1] >> 2);
-#else
- m_off += (ip[0] >> 2) + (ip[1] << 6);
-#endif
- ip += 2;
- if (m_off == 0)
- goto eof_found;
- m_off += 0x4000;
-#if defined(LZO1Z)
- last_m_off = m_off;
-#endif
-#else
-#if defined(LZO1Z)
- m_pos -= (ip[0] << 6) + (ip[1] >> 2);
-#elif (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
- m_pos -= UA_GET_LE16(ip) >> 2;
-#else
- m_pos -= (ip[0] >> 2) + (ip[1] << 6);
-#endif
- ip += 2;
- if (m_pos == op)
- goto eof_found;
- m_pos -= 0x4000;
-#if defined(LZO1Z)
- last_m_off = pd((const lzo_bytep)op, m_pos);
-#endif
-#endif
- }
- else
- {
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = 1 + (t << 6) + (*ip++ >> 2);
- last_m_off = m_off;
-#else
- m_off = 1 + (t >> 2) + (*ip++ << 2);
-#endif
- NEED_OP(2);
- t = 2; COPY_DICT(t,m_off)
-#else
-#if defined(LZO1Z)
- t = 1 + (t << 6) + (*ip++ >> 2);
- m_pos = op - t;
- last_m_off = t;
-#else
- m_pos = op - 1;
- m_pos -= t >> 2;
- m_pos -= *ip++ << 2;
-#endif
- TEST_LB(m_pos); NEED_OP(2);
- *op++ = *m_pos++; *op++ = *m_pos;
-#endif
- goto match_done;
- }
-
-#if defined(COPY_DICT)
-
- NEED_OP(t+3-1);
- t += 3-1; COPY_DICT(t,m_off)
-
-#else
-
- TEST_LB(m_pos); assert(t > 0); NEED_OP(t+3-1);
-#if (LZO_OPT_UNALIGNED64) && (LZO_OPT_UNALIGNED32)
- if (op - m_pos >= 8)
- {
- t += (3 - 1);
- if (t >= 8) do
- {
- UA_COPY8(op,m_pos);
- op += 8; m_pos += 8; t -= 8;
- } while (t >= 8);
- if (t >= 4)
- {
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4;
- }
- if (t > 0)
- {
- *op++ = m_pos[0];
- if (t > 1) { *op++ = m_pos[1]; if (t > 2) { *op++ = m_pos[2]; } }
- }
- }
- else
-#elif (LZO_OPT_UNALIGNED32) || (LZO_ALIGNED_OK_4)
-#if !(LZO_OPT_UNALIGNED32)
- if (t >= 2 * 4 - (3 - 1) && PTR_ALIGNED2_4(op,m_pos))
- {
- assert((op - m_pos) >= 4);
-#else
- if (t >= 2 * 4 - (3 - 1) && (op - m_pos) >= 4)
- {
-#endif
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4 - (3 - 1);
- do {
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4;
- } while (t >= 4);
- if (t > 0) do *op++ = *m_pos++; while (--t > 0);
- }
- else
-#endif
- {
-copy_match:
- *op++ = *m_pos++; *op++ = *m_pos++;
- do *op++ = *m_pos++; while (--t > 0);
- }
-
-#endif
-
-match_done:
-#if defined(LZO1Z)
- t = ip[-1] & 3;
-#else
- t = ip[-2] & 3;
-#endif
- if (t == 0)
- break;
-
-match_next:
- assert(t > 0); assert(t < 4); NEED_OP(t); NEED_IP(t+3);
-#if 0
- do *op++ = *ip++; while (--t > 0);
-#else
- *op++ = *ip++;
- if (t > 1) { *op++ = *ip++; if (t > 2) { *op++ = *ip++; } }
-#endif
- t = *ip++;
- }
- }
-
-eof_found:
- *out_len = pd(op, out);
- return (ip == ip_end ? LZO_E_OK :
- (ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : LZO_E_INPUT_OVERRUN));
-
-#if defined(HAVE_NEED_IP)
-input_overrun:
- *out_len = pd(op, out);
- return LZO_E_INPUT_OVERRUN;
-#endif
-
-#if defined(HAVE_NEED_OP)
-output_overrun:
- *out_len = pd(op, out);
- return LZO_E_OUTPUT_OVERRUN;
-#endif
-
-#if defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-lookbehind_overrun:
- *out_len = pd(op, out);
- return LZO_E_LOOKBEHIND_OVERRUN;
-#endif
-}
-
-#endif
-
-#define LZO_TEST_OVERRUN 1
-#undef DO_DECOMPRESS
-#define DO_DECOMPRESS lzo1x_decompress_safe
-
-#if !defined(MINILZO_CFG_SKIP_LZO1X_DECOMPRESS_SAFE)
-
-#if defined(LZO_TEST_OVERRUN)
-# if !defined(LZO_TEST_OVERRUN_INPUT)
-# define LZO_TEST_OVERRUN_INPUT 2
-# endif
-# if !defined(LZO_TEST_OVERRUN_OUTPUT)
-# define LZO_TEST_OVERRUN_OUTPUT 2
-# endif
-# if !defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-# define LZO_TEST_OVERRUN_LOOKBEHIND 1
-# endif
-#endif
-
-#undef TEST_IP
-#undef TEST_OP
-#undef TEST_IP_AND_TEST_OP
-#undef TEST_LB
-#undef TEST_LBO
-#undef NEED_IP
-#undef NEED_OP
-#undef TEST_IV
-#undef TEST_OV
-#undef HAVE_TEST_IP
-#undef HAVE_TEST_OP
-#undef HAVE_NEED_IP
-#undef HAVE_NEED_OP
-#undef HAVE_ANY_IP
-#undef HAVE_ANY_OP
-
-#if defined(LZO_TEST_OVERRUN_INPUT)
-# if (LZO_TEST_OVERRUN_INPUT >= 1)
-# define TEST_IP (ip < ip_end)
-# endif
-# if (LZO_TEST_OVERRUN_INPUT >= 2)
-# define NEED_IP(x) \
- if ((lzo_uint)(ip_end - ip) < (lzo_uint)(x)) goto input_overrun
-# define TEST_IV(x) if ((x) > (lzo_uint)0 - (511)) goto input_overrun
-# endif
-#endif
-
-#if defined(LZO_TEST_OVERRUN_OUTPUT)
-# if (LZO_TEST_OVERRUN_OUTPUT >= 1)
-# define TEST_OP (op <= op_end)
-# endif
-# if (LZO_TEST_OVERRUN_OUTPUT >= 2)
-# undef TEST_OP
-# define NEED_OP(x) \
- if ((lzo_uint)(op_end - op) < (lzo_uint)(x)) goto output_overrun
-# define TEST_OV(x) if ((x) > (lzo_uint)0 - (511)) goto output_overrun
-# endif
-#endif
-
-#if defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-# define TEST_LB(m_pos) if (PTR_LT(m_pos,out) || PTR_GE(m_pos,op)) goto lookbehind_overrun
-# define TEST_LBO(m_pos,o) if (PTR_LT(m_pos,out) || PTR_GE(m_pos,op-(o))) goto lookbehind_overrun
-#else
-# define TEST_LB(m_pos) ((void) 0)
-# define TEST_LBO(m_pos,o) ((void) 0)
-#endif
-
-#if !defined(LZO_EOF_CODE) && !defined(TEST_IP)
-# define TEST_IP (ip < ip_end)
-#endif
-
-#if defined(TEST_IP)
-# define HAVE_TEST_IP 1
-#else
-# define TEST_IP 1
-#endif
-#if defined(TEST_OP)
-# define HAVE_TEST_OP 1
-#else
-# define TEST_OP 1
-#endif
-
-#if defined(HAVE_TEST_IP) && defined(HAVE_TEST_OP)
-# define TEST_IP_AND_TEST_OP (TEST_IP && TEST_OP)
-#elif defined(HAVE_TEST_IP)
-# define TEST_IP_AND_TEST_OP TEST_IP
-#elif defined(HAVE_TEST_OP)
-# define TEST_IP_AND_TEST_OP TEST_OP
-#else
-# define TEST_IP_AND_TEST_OP 1
-#endif
-
-#if defined(NEED_IP)
-# define HAVE_NEED_IP 1
-#else
-# define NEED_IP(x) ((void) 0)
-# define TEST_IV(x) ((void) 0)
-#endif
-#if defined(NEED_OP)
-# define HAVE_NEED_OP 1
-#else
-# define NEED_OP(x) ((void) 0)
-# define TEST_OV(x) ((void) 0)
-#endif
-
-#if defined(HAVE_TEST_IP) || defined(HAVE_NEED_IP)
-# define HAVE_ANY_IP 1
-#endif
-#if defined(HAVE_TEST_OP) || defined(HAVE_NEED_OP)
-# define HAVE_ANY_OP 1
-#endif
-
-#if defined(DO_DECOMPRESS)
-LZO_PUBLIC(int)
-DO_DECOMPRESS ( const lzo_bytep in , lzo_uint in_len,
- lzo_bytep out, lzo_uintp out_len,
- lzo_voidp wrkmem )
-#endif
-{
- lzo_bytep op;
- const lzo_bytep ip;
- lzo_uint t;
-#if defined(COPY_DICT)
- lzo_uint m_off;
- const lzo_bytep dict_end;
-#else
- const lzo_bytep m_pos;
-#endif
-
- const lzo_bytep const ip_end = in + in_len;
-#if defined(HAVE_ANY_OP)
- lzo_bytep const op_end = out + *out_len;
-#endif
-#if defined(LZO1Z)
- lzo_uint last_m_off = 0;
-#endif
-
- LZO_UNUSED(wrkmem);
-
-#if defined(COPY_DICT)
- if (dict)
- {
- if (dict_len > M4_MAX_OFFSET)
- {
- dict += dict_len - M4_MAX_OFFSET;
- dict_len = M4_MAX_OFFSET;
- }
- dict_end = dict + dict_len;
- }
- else
- {
- dict_len = 0;
- dict_end = NULL;
- }
-#endif
-
- *out_len = 0;
-
- op = out;
- ip = in;
-
- NEED_IP(1);
- if (*ip > 17)
- {
- t = *ip++ - 17;
- if (t < 4)
- goto match_next;
- assert(t > 0); NEED_OP(t); NEED_IP(t+3);
- do *op++ = *ip++; while (--t > 0);
- goto first_literal_run;
- }
-
- for (;;)
- {
- NEED_IP(3);
- t = *ip++;
- if (t >= 16)
- goto match;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_IV(t);
- NEED_IP(1);
- }
- t += 15 + *ip++;
- }
- assert(t > 0); NEED_OP(t+3); NEED_IP(t+6);
-#if (LZO_OPT_UNALIGNED64) && (LZO_OPT_UNALIGNED32)
- t += 3;
- if (t >= 8) do
- {
- UA_COPY8(op,ip);
- op += 8; ip += 8; t -= 8;
- } while (t >= 8);
- if (t >= 4)
- {
- UA_COPY4(op,ip);
- op += 4; ip += 4; t -= 4;
- }
- if (t > 0)
- {
- *op++ = *ip++;
- if (t > 1) { *op++ = *ip++; if (t > 2) { *op++ = *ip++; } }
- }
-#elif (LZO_OPT_UNALIGNED32) || (LZO_ALIGNED_OK_4)
-#if !(LZO_OPT_UNALIGNED32)
- if (PTR_ALIGNED2_4(op,ip))
- {
-#endif
- UA_COPY4(op,ip);
- op += 4; ip += 4;
- if (--t > 0)
- {
- if (t >= 4)
- {
- do {
- UA_COPY4(op,ip);
- op += 4; ip += 4; t -= 4;
- } while (t >= 4);
- if (t > 0) do *op++ = *ip++; while (--t > 0);
- }
- else
- do *op++ = *ip++; while (--t > 0);
- }
-#if !(LZO_OPT_UNALIGNED32)
- }
- else
-#endif
-#endif
-#if !(LZO_OPT_UNALIGNED32)
- {
- *op++ = *ip++; *op++ = *ip++; *op++ = *ip++;
- do *op++ = *ip++; while (--t > 0);
- }
-#endif
-
-first_literal_run:
-
- t = *ip++;
- if (t >= 16)
- goto match;
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2);
- last_m_off = m_off;
-#else
- m_off = (1 + M2_MAX_OFFSET) + (t >> 2) + (*ip++ << 2);
-#endif
- NEED_OP(3);
- t = 3; COPY_DICT(t,m_off)
-#else
-#if defined(LZO1Z)
- t = (1 + M2_MAX_OFFSET) + (t << 6) + (*ip++ >> 2);
- m_pos = op - t;
- last_m_off = t;
-#else
- m_pos = op - (1 + M2_MAX_OFFSET);
- m_pos -= t >> 2;
- m_pos -= *ip++ << 2;
-#endif
- TEST_LB(m_pos); NEED_OP(3);
- *op++ = *m_pos++; *op++ = *m_pos++; *op++ = *m_pos;
-#endif
- goto match_done;
-
- for (;;) {
-match:
- if (t >= 64)
- {
-#if defined(COPY_DICT)
-#if defined(LZO1X)
- m_off = 1 + ((t >> 2) & 7) + (*ip++ << 3);
- t = (t >> 5) - 1;
-#elif defined(LZO1Y)
- m_off = 1 + ((t >> 2) & 3) + (*ip++ << 2);
- t = (t >> 4) - 3;
-#elif defined(LZO1Z)
- m_off = t & 0x1f;
- if (m_off >= 0x1c)
- m_off = last_m_off;
- else
- {
- m_off = 1 + (m_off << 6) + (*ip++ >> 2);
- last_m_off = m_off;
- }
- t = (t >> 5) - 1;
-#endif
-#else
-#if defined(LZO1X)
- m_pos = op - 1;
- m_pos -= (t >> 2) & 7;
- m_pos -= *ip++ << 3;
- t = (t >> 5) - 1;
-#elif defined(LZO1Y)
- m_pos = op - 1;
- m_pos -= (t >> 2) & 3;
- m_pos -= *ip++ << 2;
- t = (t >> 4) - 3;
-#elif defined(LZO1Z)
- {
- lzo_uint off = t & 0x1f;
- m_pos = op;
- if (off >= 0x1c)
- {
- assert(last_m_off > 0);
- m_pos -= last_m_off;
- }
- else
- {
- off = 1 + (off << 6) + (*ip++ >> 2);
- m_pos -= off;
- last_m_off = off;
- }
- }
- t = (t >> 5) - 1;
-#endif
- TEST_LB(m_pos); assert(t > 0); NEED_OP(t+3-1);
- goto copy_match;
-#endif
- }
- else if (t >= 32)
- {
- t &= 31;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_OV(t);
- NEED_IP(1);
- }
- t += 31 + *ip++;
- NEED_IP(2);
- }
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = 1 + (ip[0] << 6) + (ip[1] >> 2);
- last_m_off = m_off;
-#else
- m_off = 1 + (ip[0] >> 2) + (ip[1] << 6);
-#endif
-#else
-#if defined(LZO1Z)
- {
- lzo_uint off = 1 + (ip[0] << 6) + (ip[1] >> 2);
- m_pos = op - off;
- last_m_off = off;
- }
-#elif (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
- m_pos = op - 1;
- m_pos -= UA_GET_LE16(ip) >> 2;
-#else
- m_pos = op - 1;
- m_pos -= (ip[0] >> 2) + (ip[1] << 6);
-#endif
-#endif
- ip += 2;
- }
- else if (t >= 16)
- {
-#if defined(COPY_DICT)
- m_off = (t & 8) << 11;
-#else
- m_pos = op;
- m_pos -= (t & 8) << 11;
-#endif
- t &= 7;
- if (t == 0)
- {
- while (*ip == 0)
- {
- t += 255;
- ip++;
- TEST_OV(t);
- NEED_IP(1);
- }
- t += 7 + *ip++;
- NEED_IP(2);
- }
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off += (ip[0] << 6) + (ip[1] >> 2);
-#else
- m_off += (ip[0] >> 2) + (ip[1] << 6);
-#endif
- ip += 2;
- if (m_off == 0)
- goto eof_found;
- m_off += 0x4000;
-#if defined(LZO1Z)
- last_m_off = m_off;
-#endif
-#else
-#if defined(LZO1Z)
- m_pos -= (ip[0] << 6) + (ip[1] >> 2);
-#elif (LZO_OPT_UNALIGNED16) && (LZO_ABI_LITTLE_ENDIAN)
- m_pos -= UA_GET_LE16(ip) >> 2;
-#else
- m_pos -= (ip[0] >> 2) + (ip[1] << 6);
-#endif
- ip += 2;
- if (m_pos == op)
- goto eof_found;
- m_pos -= 0x4000;
-#if defined(LZO1Z)
- last_m_off = pd((const lzo_bytep)op, m_pos);
-#endif
-#endif
- }
- else
- {
-#if defined(COPY_DICT)
-#if defined(LZO1Z)
- m_off = 1 + (t << 6) + (*ip++ >> 2);
- last_m_off = m_off;
-#else
- m_off = 1 + (t >> 2) + (*ip++ << 2);
-#endif
- NEED_OP(2);
- t = 2; COPY_DICT(t,m_off)
-#else
-#if defined(LZO1Z)
- t = 1 + (t << 6) + (*ip++ >> 2);
- m_pos = op - t;
- last_m_off = t;
-#else
- m_pos = op - 1;
- m_pos -= t >> 2;
- m_pos -= *ip++ << 2;
-#endif
- TEST_LB(m_pos); NEED_OP(2);
- *op++ = *m_pos++; *op++ = *m_pos;
-#endif
- goto match_done;
- }
-
-#if defined(COPY_DICT)
-
- NEED_OP(t+3-1);
- t += 3-1; COPY_DICT(t,m_off)
-
-#else
-
- TEST_LB(m_pos); assert(t > 0); NEED_OP(t+3-1);
-#if (LZO_OPT_UNALIGNED64) && (LZO_OPT_UNALIGNED32)
- if (op - m_pos >= 8)
- {
- t += (3 - 1);
- if (t >= 8) do
- {
- UA_COPY8(op,m_pos);
- op += 8; m_pos += 8; t -= 8;
- } while (t >= 8);
- if (t >= 4)
- {
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4;
- }
- if (t > 0)
- {
- *op++ = m_pos[0];
- if (t > 1) { *op++ = m_pos[1]; if (t > 2) { *op++ = m_pos[2]; } }
- }
- }
- else
-#elif (LZO_OPT_UNALIGNED32) || (LZO_ALIGNED_OK_4)
-#if !(LZO_OPT_UNALIGNED32)
- if (t >= 2 * 4 - (3 - 1) && PTR_ALIGNED2_4(op,m_pos))
- {
- assert((op - m_pos) >= 4);
-#else
- if (t >= 2 * 4 - (3 - 1) && (op - m_pos) >= 4)
- {
-#endif
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4 - (3 - 1);
- do {
- UA_COPY4(op,m_pos);
- op += 4; m_pos += 4; t -= 4;
- } while (t >= 4);
- if (t > 0) do *op++ = *m_pos++; while (--t > 0);
- }
- else
-#endif
- {
-copy_match:
- *op++ = *m_pos++; *op++ = *m_pos++;
- do *op++ = *m_pos++; while (--t > 0);
- }
-
-#endif
-
-match_done:
-#if defined(LZO1Z)
- t = ip[-1] & 3;
-#else
- t = ip[-2] & 3;
-#endif
- if (t == 0)
- break;
-
-match_next:
- assert(t > 0); assert(t < 4); NEED_OP(t); NEED_IP(t+3);
-#if 0
- do *op++ = *ip++; while (--t > 0);
-#else
- *op++ = *ip++;
- if (t > 1) { *op++ = *ip++; if (t > 2) { *op++ = *ip++; } }
-#endif
- t = *ip++;
- }
- }
-
-eof_found:
- *out_len = pd(op, out);
- return (ip == ip_end ? LZO_E_OK :
- (ip < ip_end ? LZO_E_INPUT_NOT_CONSUMED : LZO_E_INPUT_OVERRUN));
-
-#if defined(HAVE_NEED_IP)
-input_overrun:
- *out_len = pd(op, out);
- return LZO_E_INPUT_OVERRUN;
-#endif
-
-#if defined(HAVE_NEED_OP)
-output_overrun:
- *out_len = pd(op, out);
- return LZO_E_OUTPUT_OVERRUN;
-#endif
-
-#if defined(LZO_TEST_OVERRUN_LOOKBEHIND)
-lookbehind_overrun:
- *out_len = pd(op, out);
- return LZO_E_LOOKBEHIND_OVERRUN;
-#endif
-}
-
-#endif
-
-/***** End of minilzo.c *****/
diff --git a/contrib/libs/minilzo/minilzo.h b/contrib/libs/minilzo/minilzo.h
deleted file mode 100644
index 9fc11304b2..0000000000
--- a/contrib/libs/minilzo/minilzo.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/* minilzo.h -- mini subset of the LZO real-time data compression library
-
- This file is part of the LZO real-time data compression library.
-
- Copyright (C) 1996-2015 Markus Franz Xaver Johannes Oberhumer
- All Rights Reserved.
-
- The LZO library is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of
- the License, or (at your option) any later version.
-
- The LZO library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with the LZO library; see the file COPYING.
- If not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
- Markus F.X.J. Oberhumer
- <markus@oberhumer.com>
- http://www.oberhumer.com/opensource/lzo/
- */
-
-/*
- * NOTE:
- * the full LZO package can be found at
- * http://www.oberhumer.com/opensource/lzo/
- */
-
-
-#ifndef __MINILZO_H_INCLUDED
-#define __MINILZO_H_INCLUDED 1
-
-/* This is Arcadia-specific header used for symbol mangling. */
-#include "protect.h"
-
-#define MINILZO_VERSION 0x2090
-
-#if defined(__LZOCONF_H_INCLUDED)
-# error "you cannot use both LZO and miniLZO"
-#endif
-
-/* internal Autoconf configuration file - only used when building miniLZO */
-#ifdef MINILZO_HAVE_CONFIG_H
-#error # include <config.h>
-#endif
-#include <limits.h>
-#include <stddef.h>
-
-#ifndef __LZODEFS_H_INCLUDED
-#include "lzodefs.h"
-#endif
-#undef LZO_HAVE_CONFIG_H
-#include "lzoconf.h"
-
-#if !defined(LZO_VERSION) || (LZO_VERSION != MINILZO_VERSION)
-# error "version mismatch in header files"
-#endif
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-/***********************************************************************
-//
-************************************************************************/
-
-/* Memory required for the wrkmem parameter.
- * When the required size is 0, you can also pass a NULL pointer.
- */
-
-#define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS
-#define LZO1X_1_MEM_COMPRESS ((lzo_uint32_t) (16384L * lzo_sizeof_dict_t))
-#define LZO1X_MEM_DECOMPRESS (0)
-
-
-/* compression */
-LZO_EXTERN(int)
-lzo1x_1_compress ( const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem );
-
-/* decompression */
-LZO_EXTERN(int)
-lzo1x_decompress ( const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem /* NOT USED */ );
-
-/* safe decompression with overrun testing */
-LZO_EXTERN(int)
-lzo1x_decompress_safe ( const lzo_bytep src, lzo_uint src_len,
- lzo_bytep dst, lzo_uintp dst_len,
- lzo_voidp wrkmem /* NOT USED */ );
-
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* already included */
-
-
-/* vim:set ts=4 sw=4 et: */
diff --git a/contrib/libs/minilzo/protect.h b/contrib/libs/minilzo/protect.h
deleted file mode 100644
index 9eeae941aa..0000000000
--- a/contrib/libs/minilzo/protect.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#pragma once
-
-#define __lzo_align_gap _minilzo___lzo_align_gap
-#define __lzo_init_v2 _minilzo___lzo_init_v2
-#define __lzo_ptr_linear _minilzo___lzo_ptr_linear
-#define _lzo_config_check _minilzo__lzo_config_check
-#define _lzo_version_date _minilzo__lzo_version_date
-#define _lzo_version_string _minilzo__lzo_version_string
-#define lzo1x_1_compress _minilzo_lzo1x_1_compress
-#define lzo1x_decompress _minilzo_lzo1x_decompress
-#define lzo1x_decompress_safe _minilzo_lzo1x_decompress_safe
-#define lzo_adler32 _minilzo_lzo_adler32
-#define lzo_copyright _minilzo_lzo_copyright
-#define lzo_crc32 _minilzo_lzo_crc32
-#define lzo_get_crc32_table _minilzo_lzo_get_crc32_table
-#define lzo_version _minilzo_lzo_version
-#define lzo_version_date _minilzo_lzo_version_date
-#define lzo_version_string _minilzo_lzo_version_string
diff --git a/kikimr/.gitignore b/kikimr/.gitignore
deleted file mode 100644
index 504ed2bc3b..0000000000
--- a/kikimr/.gitignore
+++ /dev/null
@@ -1,17 +0,0 @@
-*.pyc
-*.so
-.om.cache
-om.conf.local
-local.cmake
-local.ymake
-/.gcc-compiler
-/.gxx-compiler
-.svn
-*.log
-test-results
-*.a
-*_pb2.py
-*.pkg
-*.so.1
-kikimr/driver/kikimr
-.idea
diff --git a/kikimr/.kikimr.root b/kikimr/.kikimr.root
deleted file mode 100644
index 0fe6c75b88..0000000000
--- a/kikimr/.kikimr.root
+++ /dev/null
@@ -1 +0,0 @@
-KiKiMR project root \ No newline at end of file
diff --git a/kikimr/README.md b/kikimr/README.md
deleted file mode 100644
index a0c453aa1d..0000000000
--- a/kikimr/README.md
+++ /dev/null
@@ -1,40 +0,0 @@
-# Yandex Database (YDB aka KiKiMR) and Related Projects
-
-See <https://wiki.yandex-team.ru/kikimr> for details.
-
-> For client SDK, examples, API description go to the `public` directory.
-
-## YDB Directory Structure
-
-`blockstore` -- a contract for Blockstore project (messages and their ids);
-`core` -- YDB implementation;
-`docs` -- documentation as a code;
-`driver` -- YDB binary for Yandex (has some internal deps like Unified Agent, TVM, etc);
-`driver_lib` -- a common part for `driver` and `driver_oss`;
-`driver_oss` -- YDB binary for open source;
-`filestore` -- a contract for Filestore project (messages and their ids);
-`kikhouse_new` -- ClickHouse over YDB;
-`library` -- standalone libraries used by YDB;
-`mvp` -- a proxy to YDB, implements UI calls, etc;
-`papers` -- papers about YDB;
-`persqueue` -- persistent queue (aka Logbroker) implementation;
-`public` -- SDKs, APIs and tools for YDB end users;
-`services` -- grpc services for YDB public API;
-`sqs` -- Amazon SQS (Simple Queue Service) API implementation on top of YDB. It is provided in Yandex.Cloud as
-YMQ (Yandex Message Queue);
-`streaming` -- YQL Streams implementation (as a part of Yandex Query service);
-`ydbcp` -- YDB Control Plane;
-`yf` -- Yandex Functions serverless computing and job execution service;
-`yq` -- Yandex Query implementation;
-
-
-## Rest
-ci
-deployment
-juggler
-production
-scripts
-serverless_proxy
-testing
-tools
-
diff --git a/kikimr/a.yaml b/kikimr/a.yaml
deleted file mode 100644
index ae35b112e3..0000000000
--- a/kikimr/a.yaml
+++ /dev/null
@@ -1,532 +0,0 @@
-service: kikimr
-title: CI for YDB
-ci:
- secret: sec-01ekjvt2kbg4ag64z9z03jt0hp
- runtime:
- sandbox-owner: STORAGE-DEV
- actions:
- run_sdk_java_tests:
- title: Run Java tests using Maven
- flow: run_java_tests
- triggers:
- - on: pr
- filters:
- - sub-paths: [ public/sdk/java/** ]
- deploy_sdk_java_artifacts:
- title: Deploy Java SDK artifacts
- flow: deploy_java_sdk_to_internal
- triggers:
- - on: commit
- filters:
- - sub-paths: [ public/sdk/java/pom.xml ]
- release-title-source: flow
- releases:
- run_periodic_export_of_data:
- title: Run periodic export of data to monitoring
- flow: run_periodic_export_of_data
- auto:
- conditions:
- - schedule:
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Run periodic export of data to monitoring
- sync_python_sdk_from_git:
- title: Sync from git
- flow: sync_python_sdk_from_git
- auto:
- conditions:
- - schedule:
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Sync from git
- publish_ydb_internal:
- title: Build YDB Python SDK (Yandex Internal)
- flow: publish_python_sdk_pypi_internal
- auto: true
- flow-vars:
- package-path: kikimr/public/sdk/packages/python/pkg.json
- stages:
- package:
- title: Build Python SDK
- publish_ydb_persqueue_internal:
- title: Build YDB Persqueue Python SDK (Yandex Internal)
- flow: publish_python_sdk_pypi_internal
- auto: true
- flow-vars:
- package-path: kikimr/public/sdk/packages/ydb-persqueue/pkg.json
- stages:
- package:
- title: Build Python SDK
- ydb_server_recipe_binary:
- title: YDB Recipe Binary Release
- description: |
- Релиз бинарника YDB сервера в Аркадийный рецептах.
-
- По умолчанию, ресурс с бинарником собирается автоматически из HEAD trunk.
- После сборки ресурсов создается PR и мержится после успешного прохождения тестов.
- Чтобы переключить рецепт на новую мажорную версию, нужно обновить ветку в [a.yaml](http://a.yandex-team.ru/arc_vcs/kikimr/a.yaml), параметр `checkout-arcadia-from-url`.
-
- Автоматические релизы по понедельникам в первой половине дня.
-
- flow: recipe_tool
- flow-vars:
- build-project: kikimr/driver/kikimr
- prebuilt-ya-make-path: kikimr/public/tools/package/stable
- checkout-arcadia-from-url: "arcadia-arc:/#trunk"
- auto:
- conditions:
- - schedule:
- days: MON
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Build binaries
- lbk_recipe_tool:
- title: LOGBROKER Recipe Tool Release
- description: |
- Релиз программы, запускающей сервер Аркадийном рецепте LOGBROKER.
-
- По умолчанию, рецепт обновляеся автоматически из trunk.
- Релизы программы утром в понедельник.
- flow: recipe_tool
- flow-vars:
- build-project: kikimr/public/tools/lbk_recipe/bin/lbk_recipe
- prebuilt-ya-make-path: kikimr/public/tools/lbk_recipe/prebuilt
- checkout-arcadia-from-url: "arcadia-arc:/#trunk"
- auto:
- conditions:
- - schedule:
- days: MON
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Build binaries
- sqs_recipe_tool:
- title: SQS Recipe Tool Release
- description: |
- Релиз программы, запускающей сервер Аркадийном рецепте SQS.
-
- По умолчанию, рецепт обновляеся автоматически из trunk.
- Релизы программы утром в понедельник.
- flow: recipe_tool
- flow-vars:
- build-project: kikimr/public/tools/sqs_recipe/bin/sqs_recipe
- prebuilt-ya-make-path: kikimr/public/tools/sqs_recipe/prebuilt
- checkout-arcadia-from-url: "arcadia-arc:/#trunk"
- auto:
- conditions:
- - schedule:
- days: MON
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Build binaries
- ydb_recipe_tool:
- title: YDB Recipe Tool Release
- flow: recipe_tool
- description: |
- Релиз программы, запускающей сервер Аркадийном рецепте.
-
- По умолчанию, рецепт обновляеся автоматически из trunk.
- Подробнее в [a.yaml](http://a.yandex-team.ru/arc_vcs/kikimr/a.yaml)
- flow-vars:
- build-project: kikimr/public/tools/ydb_recipe/bin/ydb_recipe
- prebuilt-ya-make-path: kikimr/public/tools/ydb_recipe/prebuilt
- checkout-arcadia-from-url: "arcadia-arc:/#trunk"
- auto:
- conditions:
- - schedule:
- days: MON
- time: 9:00 - 12:00 MSK
- since-last-release: 24h
- stages:
- package:
- title: Build binaries
- ydbcp_release:
- title: YDB Control Plane Release
- description: |
- Релиз YDB Control Plane
- flow: ydbcp-release-flow
- start-version: 50
- branches:
- pattern: releases/ydb/ydbcp/stable-${version}
- auto-create: true
- stages:
- - id: test-ydbcp
- title: Run ydbcp tests
- - id: build-ydbcp
- title: Build ydbcp
- flows:
- run_java_tests:
- jobs:
- run_test:
- title: Build and mvn install kikimr/public/sdk/java
- task: common/misc/run_command
- requirements:
- cores: 16
- ram: 32 GB
- sandbox:
- client_tags: GENERIC & LINUX & SSD
- container_resource: 2309143241 # Container with Java 8, Java 11 and Maven
- dns: dns64
- input:
- config:
- cmd_line: |
- set -ex
- cd $ARCADIA_PATH
- export PATH=$ARCADIA_PATH:$PATH
- export JAVA_HOME=/usr/local/java8
- mkdir -p ~/.m2
- cp ci/tasklet/registry/common/misc/run_command/maven/settings.xml ~/.m2/
- cd kikimr/public/sdk/java
- ya make
- ./gen-proto.sh
- mvn -B -Pwith-examples install
- secret_environment_variables:
- - key: YA_TOKEN
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: ci.token
- arc_mount_config:
- enabled: true
- deploy_java_sdk_to_internal:
- jobs:
- deploy_all:
- title: Build and deploy kikimr/public/sdk/java artefacts to internal repo
- task: common/misc/run_command
- requirements:
- cores: 16
- ram: 32 GB
- sandbox:
- client_tags: GENERIC & LINUX & SSD
- container_resource: 2309143241 # Container with Java 8, Java 11 and Maven
- dns: dns64
- input:
- config:
- cmd_line: |
- set -ex
- cd $ARCADIA_PATH
- export PATH=$ARCADIA_PATH:$PATH
- export JAVA_HOME=/usr/local/java8
- mkdir -p ~/.m2
- cp ci/tasklet/registry/common/misc/run_command/maven/settings.xml ~/.m2/
- sed 's#[ ]</servers>#<server><id>internal</id><username>$${env.USERNAME}</username><password>$${env.PASSWORD}</password></server></servers>#' -i ~/.m2/settings.xml
- cd kikimr/public/sdk/java
- ya make
- ./gen-proto.sh
- mvn -B -Pinternal install deploy
- secret_environment_variables:
- - key: USERNAME
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: yandex.infra.username
- - key: PASSWORD
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: yandex.infra.password
- arc_mount_config:
- enabled: true
- run_periodic_export_of_data:
- jobs:
- run_test:
- title: run_periodic_export_of_data
- task: common/misc/run_command
- requirements:
- sandbox:
- client_tags: GENERIC & LINUX
- dns: dns64
- input:
- config:
- secret_environment_variables:
- - key: YC_OAUTH_TOKEN
- secret_spec:
- uuid: sec-01fzd4egdm6zk7bfxd3bnf3fnw
- key: token
- - key: YA_TOKEN
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: ci.token
- - key: YT_TOKEN
- secret_spec:
- uuid: sec-01fzd4egdm6zk7bfxd3bnf3fnw
- key: yt
- arc_mount_config:
- enabled: true
- cmd_line: |
- set -xe
- curl https://s3.mds.yandex.net/mcdev/ycp/install.sh | bash
- export PATH=/home/sandbox/ycp/bin:$PATH
- ycp init
- cp $ARCADIA_PATH/kikimr/ydbcp/abc_database/config.yaml /home/sandbox/.config/ycp/config.yaml
- sed -i "s/TOKEN_TO_REPLACE/"$YC_OAUTH_TOKEN"/g" /home/sandbox/.config/ycp/config.yaml
- ycp --format json --profile internal team integration abc resolve -r - <<<"abc_slug: kikimr"
- cd $ARCADIA_PATH/kikimr/ydbcp/abc_database/
- $ARCADIA_PATH/ya make -r
- ./collect_info
- cat result.txt
- cat result.txt | YT_PROXY=hahn $ARCADIA_PATH/ya tool yt write --table //home/solomon/service_provider_alerts/service_provider_exports/ydb --format json
- run_build_from_git:
- jobs:
- run_test:
- title: Build python sdk and then run tests
- task: common/misc/run_command
- requirements:
- cores: 56
- ram: 256 GB
- sandbox:
- client_tags: GENERIC & LINUX & SSD
- dns: dns64
- input:
- config:
- result_resources:
- - path: package
- description: YDBD TAR
- type: OTHER_RESOURCE
- compression_type: tgz
- ci_badge: true
- attributes:
- released: stable
- ttl: "14"
- cmd_line: |
- git clone https://github.yandex-team.ru/arcadia-devtools/ydb_oss.git
- mkdir $RESULT_RESOURCES_PATH/package
- cd ydb_oss
- ./ya make -r -k ydb/apps/ydbd/
- cp ydb/apps/ydbd/ydbd $RESULT_RESOURCES_PATH/package/
- cp ydb/apps/ydbd/*.so $RESULT_RESOURCES_PATH/package/
- publish_python_sdk_pypi_internal:
- title: Build YDB Python SDK (Yandex Internal)
- jobs:
- package:
- title: Build YDB Python SDK (Yandex Internal)
- description: Runs ya package in sandbox
- task: common/arcadia/ya_package
- input:
- packages: ${flow-vars.package-path}
- build_type: release
- package_type: wheel
- publish_package: True
- publish_to: ""
- wheel_access_key_token: robot_ydb_pypi_access_token
- wheel_secret_key_token: robot_ydb_pypi_key_token
- wheel_upload_repo: https://pypi.yandex-team.ru/simple
- build_system: ya
- sandbox_container: 1329390397
- stage: package
- ydbcp-release-flow:
- title: Release YDBCP
- jobs:
- start-release:
- title: Start release
- task: dummy
- stage: test-ydbcp
- run-ydbcp-tests:
- title: Run ydbcp tests
- description: Run ydbcp tests
- task: common/arcadia/ya_make
- stage: test-ydbcp
- needs: start-release
- requirements:
- sandbox:
- client_tags: GENERIC & LINUX & SSD
- cores: 56
- ram: 256GB
- input: &run-tests-on-distbuild-input
- build_system: ya_force
- test: True
- keep_on: True
- checkout: False
- checkout_mode: manual
- output_only_tests: True
- test_log_level: debug
- collect_test_cores: False
- do_not_restart: True
- strip_binaries: True
- check_return_code: True
- fail_on_any_error: True
- junit_report: True
- kill_timeout: 14400
- build_output_ttl: 1
- build_output_html_ttl: 14
- ya_timeout: 13500
- test_threads: 50
- # distbuild_pool: "//man/users/ydb"
- # env_vars: "YA_TOKEN='$(vault:value:STORAGE-DEV:ARC_TOKEN)'"
- targets: "kikimr/ydbcp"
- validate-test-failures:
- title: Validate test fails
- task: dummy
- manual:
- enabled: true
- prompt: "Ignore test failures?"
- needs: start-release
- run-create-tag:
- title: Make YDBCP release tag
- description: "Mount ARC and create the tag"
- task: common/misc/run_command
- requirements:
- cores: 2
- ram: 16G
- sandbox:
- client_tags: GENERIC & LINUX
- needs-type: any
- needs:
- - run-ydbcp-tests
- - validate-test-failures
- stage: build-ydbcp
- input:
- config:
- arc_mount_config:
- enabled: true
- revision_hash: trunk
- environment_variables:
- - key: ARC_TAG
- value: "tags/releases/ydb/ydbcp/stable-${context.version_info.major}-${not_null(context.version_info.minor, `0`)}"
- - key: ARC_COMMIT
- value: "${context.target_revision.hash}"
- secret_environment_variables:
- - key: YA_TOKEN
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: ci.token
- cmd_line: |
- pwd
- $ARC_BIN status
- $ARC_BIN push $ARC_COMMIT:$ARC_TAG
- build-ydbcp:
- title: Run build YDBCP packages
- description: Run build YDB packages
- task: common/arcadia/ya_package_2
- needs: run-create-tag
- stage: build-ydbcp
- requirements:
- sandbox:
- client_tags: GENERIC & LINUX & SSD
- cores: 56
- ram: 256GB
- input:
- build_system: ya_force
- checkout_arcadia_from_url: "arcadia-arc:/#tags/releases/ydb/ydbcp/stable-${context.version_info.major}-${not_null(context.version_info.minor, `0`)}"
- key_user: robot-simcity
- publish_package: True
- multiple_publish: True
- publish_to: search
- multiple_publish_to: search
- build_logs_ttl: 1
- package_ttl: 1
- package_type: debian
- build_output_ttl: 1
- packages: "kikimr/ydbcp/packages/ydb-ydbcp-bin/pkg.json;kikimr/ydbcp/packages/ydb-ydbcp-init/pkg.json;kikimr/ydbcp/packages/ydb-ydbcp-prestable-meta-cfg/pkg.json;kikimr/ydbcp/packages/ydb-ydbcp-prod-meta-cfg/pkg.json;"
- sync_python_sdk_from_git:
- title: Sync from git
- jobs:
- sync_ydb_python_sdk_from_git:
- title: Make PR with update
- description: Runs
- task: common/misc/run_command
- requirements:
- cores: 2
- ram: 16G
- sandbox:
- client_tags: GENERIC & LINUX
- dns: dns64
- input:
- config:
- arc_mount_config:
- enabled: true
- revision_hash: trunk
- environment_variables:
- - key: ARC_BRANCH
- value: "users/robot-kikimr-dev/execute-sync-from-github-${context.version_info.major}"
- secret_environment_variables:
- - key: YA_TOKEN
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: ci.token
- cmd_line: |
- pwd
- $ARC_BIN status
- $ARC_BIN info --json
- $ARC_BIN checkout -b execute-sync-from-github
- git clone https://github.com/yandex-cloud/ydb-python-sdk ~/tmp
- rm -rf ydb/public/sdk/python/*
- cp -r ~/tmp/* ydb/public/sdk/python/
- python3 kikimr/public/sdk/python/make_ya_make.py
- $ARC_BIN add -f ydb/public/sdk/python
- $ARC_BIN commit -am "Release YDB recipe binaries"
- $ARC_BIN push --set-upstream $ARC_BRANCH
- $ARC_BIN pr create --publish -m 'Sync YDB SDK from github' --no-edit
- recipe_tool:
- title: Recipe tool binary
- jobs:
- start-build:
- title: Build applications
- task: dummy
- stage: package
- build-project:
- title: Build Recipe Tool
- description: Runs sandbox task
- task: common/arcadia/build_arcadia_project_for_all
- needs:
- - start-build
- input:
- project: ${flow-vars.build-project}
- checkout_arcadia_from_url: ${flow-vars.checkout-arcadia-from-url}
- platforms_list:
- - linux
- - darwin
- strip_binaries: True
- definition_flags: -DUSE_YDB_TRUNK_RECIPE_TOOLS=1
- recipe_pr:
- title: Make PR with resources
- description: Runs
- task: common/misc/run_command
- requirements:
- cores: 2
- ram: 16G
- sandbox:
- client_tags: GENERIC & LINUX
- needs:
- - start-build
- - build-project
- input:
- config:
- arc_mount_config:
- enabled: true
- revision_hash: trunk
- environment_variables:
- - key: YDB_LINUX
- value: ${to_string((tasks.build-project.resources[?type == 'ARCADIA_PROJECT_TGZ' && attributes.platform == 'linux'])[0].id)}
- - key: YDB_DARWIN
- value: ${to_string((tasks.build-project.resources[?type == 'ARCADIA_PROJECT_TGZ' && attributes.platform == 'darwin'])[0].id)}
- - key: RECIPE_TOOL_PATH
- value: ${flow-vars.prebuilt-ya-make-path}
- secret_environment_variables:
- - key: YA_TOKEN
- secret_spec:
- uuid: sec-01ekjvt2kbg4ag64z9z03jt0hp
- key: ci.token
- cmd_line: |
- pwd
- $ARC_BIN status
- $ARC_BIN info --json
- $ARC_BIN checkout -b release-recipe-binaries-$YDB_LINUX-$YDB_DARWIN
- rm $RECIPE_TOOL_PATH/ya.make
- cp $RECIPE_TOOL_PATH/ya.make-template $RECIPE_TOOL_PATH/ya.make
- sed -i 's/YDB_LINUX/'$YDB_LINUX'/g' $RECIPE_TOOL_PATH/ya.make
- sed -i 's/YDB_DARWIN/'$YDB_DARWIN'/g' $RECIPE_TOOL_PATH/ya.make
- $ARC_BIN add $RECIPE_TOOL_PATH/ya.make
- $ARC_BIN commit -am "Release YDB recipe binaries"
- $ARC_BIN push --set-upstream users/robot-kikimr-dev/release-recipe-binaries-$YDB_LINUX-$YDB_DARWIN
- $ARC_BIN pr create --publish -m 'Release YDB recipe binaries' --no-edit
- autocheck:
- strong: true
- fast-targets:
- kikimr
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt
deleted file mode 100644
index 44cb2496ed..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt
+++ /dev/null
@@ -1,60 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(deprecated-cpp-v2)
-target_link_libraries(deprecated-cpp-v2 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- contrib-libs-protobuf
- cpp-client-iam
- cpp-client-ydb_persqueue
- api-grpc-yndx
- api-protos-yndx
- yndx-persqueue-read_batch_converter
- library-cpp-blockcodecs
- cpp-containers-disjoint_interval_tree
- cpp-containers-intrusive_rb_tree
- cpp-grpc-common
- cpp-http-simple
- library-cpp-json
- library-cpp-logger
- cpp-streams-lzop
- cpp-streams-zstd
- cpp-string_utils-quote
- cpp-threading-future
- cpp-tvmauth-client
- library-persqueue-topic_parser_public
- cpp-client-resources
-)
-target_sources(deprecated-cpp-v2 PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp
-)
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h
deleted file mode 100644
index 623a293dc3..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h
+++ /dev/null
@@ -1,60 +0,0 @@
-#pragma once
-#include "fake_actor.h"
-
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/actorid.h>
-#include <library/cpp/actors/core/actorsystem.h>
-#include <library/cpp/actors/core/events.h>
-
-namespace NPersQueue {
-
-struct IActorInterface {
- virtual ~IActorInterface() = default;
-
- // Sender of all messages from PQLib objects.
- virtual NActors::TActorId GetActorID() const noexcept = 0;
-};
-
-class TActorHolder : public IActorInterface
-{
-protected:
- TActorHolder(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID)
- : ActorSystem(actorSystem)
- , ParentActorID(parentActorID)
- {
- ActorID = ActorSystem->Register(new TFakeActor());
- }
-
- ~TActorHolder() {
- ActorSystem->Send(new NActors::IEventHandle(ActorID, ActorID, new NActors::TEvents::TEvPoisonPill()));
- }
-
- template <class TResponseEvent>
- void Subscribe(ui64 requestId, NThreading::TFuture<typename TResponseEvent::TResponseType>&& future) {
- auto handler = [
- requestId = requestId,
- actorSystem = ActorSystem,
- actorID = ActorID,
- parentActorID = ParentActorID
- ](const NThreading::TFuture<typename TResponseEvent::TResponseType>& future) {
- actorSystem->Send(new NActors::IEventHandle(
- parentActorID,
- actorID,
- new TResponseEvent(typename TResponseEvent::TResponseType(future.GetValue()), requestId)
- ));
- };
- future.Subscribe(handler);
- }
-
-public:
- NActors::TActorId GetActorID() const noexcept override {
- return ActorID;
- }
-
-protected:
- NActors::TActorSystem* ActorSystem;
- NActors::TActorId ParentActorID;
- NActors::TActorId ActorID;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h
deleted file mode 100644
index ce2f7682c1..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h
+++ /dev/null
@@ -1,108 +0,0 @@
-#pragma once
-#include "actor_interface.h"
-#include "responses.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-#include <util/datetime/base.h>
-
-namespace NPersQueue {
-
-template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId>
-class TProducerActorWrapper: public TActorHolder {
-public:
- using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>;
- using TCreateResponseEvent = TPQLibResponseEvent<TProducerCreateResponse, TCreateResponseEventId>;
- using TCommitResponseEvent = TPQLibResponseEvent<TProducerCommitResponse, TCommitResponseEventId>;
-
-public:
- TProducerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProducer> producer)
- : TActorHolder(actorSystem, parentActorID)
- , Producer(std::move(producer))
- {
- }
-
- void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept {
- Subscribe<TCreateResponseEvent>(requestId, Producer->Start(deadline));
- Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Producer->IsDead());
- }
-
- void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept {
- Start(TInstant::Now() + timeout, requestId, isDeadRequestId);
- }
-
- void Write(TProducerSeqNo seqNo, TData data, ui64 requestId = 0) noexcept {
- Subscribe<TCommitResponseEvent>(requestId, Producer->Write(seqNo, std::move(data)));
- }
-
- void Write(TData data, ui64 requestId = 0) noexcept {
- Subscribe<TCommitResponseEvent>(requestId, Producer->Write(std::move(data)));
- }
-
-private:
- THolder<IProducer> Producer;
-};
-
-template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId>
-class TConsumerActorWrapper: public TActorHolder {
-public:
- using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>;
- using TCreateResponseEvent = TPQLibResponseEvent<TConsumerCreateResponse, TCreateResponseEventId>;
- using TGetMessageEvent = TPQLibResponseEvent<TConsumerMessage, TGetMessageEventId>;
-
-public:
- TConsumerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IConsumer> consumer)
- : TActorHolder(actorSystem, parentActorID)
- , Consumer(std::move(consumer))
- {
- }
-
- void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept {
- Subscribe<TCreateResponseEvent>(requestId, Consumer->Start(deadline));
- Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Consumer->IsDead());
- }
-
- void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept {
- Start(TInstant::Now() + timeout, requestId, isDeadRequestId);
- }
-
- void GetNextMessage(ui64 requestId = 0) noexcept {
- Subscribe<TGetMessageEvent>(requestId, Consumer->GetNextMessage());
- }
-
- void Commit(const TVector<ui64>& cookies) noexcept {
- Consumer->Commit(cookies); // no future in PQLib API
- }
-
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- Consumer->RequestPartitionStatus(topic, partition, generation); // no future in PQLib API
- }
-
-private:
- THolder<IConsumer> Consumer;
-};
-
-template <ui32 TOriginDataEventId>
-class TProcessorActorWrapper: public TActorHolder {
-public:
- using TOriginDataEvent = TPQLibResponseEvent<TOriginData, TOriginDataEventId>;
-
-public:
- TProcessorActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProcessor> processor)
- : TActorHolder(actorSystem, parentActorID)
- , Processor(std::move(processor))
- {
- }
-
- void GetNextData(ui64 requestId = 0) noexcept {
- Subscribe<TOriginDataEvent>(requestId, Processor->GetNextData());
- }
-
-private:
- THolder<IProcessor> Processor;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp
deleted file mode 100644
index 1e08597ec3..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-#include "actor_wrappers.h"
-#include "responses.h"
-
-#include <ydb/core/testlib/basics/runtime.h>
-#include <ydb/core/testlib/basics/appdata.h>
-
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/testing/gmock_in_unittest/gmock.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/thread.h>
-
-using namespace testing;
-
-namespace NPersQueue {
-
-namespace TEvPQLibTests {
- enum EEv {
- EvPQLibObjectIsDead = EventSpaceBegin(NActors::TEvents::ES_PRIVATE),
- EvPQLibProducerCreateResponse,
- EvPQLibProducerCommitResponse,
- EvEnd
- };
-} // TEvPQLibTests
-
-class TMockProducer: public IProducer {
-public:
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override {
- return NThreading::MakeFuture<TProducerCreateResponse>(MockStart());
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData data) noexcept override {
- return Write(std::move(data));
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override {
- return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data));
- }
-
- NThreading::TFuture<TError> IsDead() noexcept override {
- return MockIsDead();
- }
-
- MOCK_METHOD(TProducerCreateResponse, MockStart, (), ());
- MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ());
- MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ());
-};
-
-Y_UNIT_TEST_SUITE(TProducerActorWrapperTest) {
- Y_UNIT_TEST(PassesEvents) {
- NActors::TTestActorRuntime runtime;
- runtime.Initialize(NKikimr::TAppPrepare().Unwrap());
- auto edgeActorId = runtime.AllocateEdgeActor();
- THolder<TMockProducer> producerHolder = MakeHolder<TMockProducer>();
- TMockProducer& producer = *producerHolder;
-
- // expectations
- TWriteResponse startResponse;
- startResponse.MutableInit()->SetMaxSeqNo(42);
- EXPECT_CALL(producer, MockStart())
- .WillOnce(Return(TProducerCreateResponse(std::move(startResponse))));
-
- NThreading::TPromise<TError> deadPromise = NThreading::NewPromise<TError>();
- EXPECT_CALL(producer, MockIsDead())
- .WillOnce(Return(deadPromise.GetFuture()));
-
- TWriteResponse writeResponse;
- writeResponse.MutableAck()->SetSeqNo(100);
- TData data("data");
- EXPECT_CALL(producer, MockWrite(data))
- .WillOnce(Return(TProducerCommitResponse(100, data, std::move(writeResponse))));
-
- // wrapper creation
- using TProducerType = TProducerActorWrapper<
- TEvPQLibTests::EvPQLibObjectIsDead,
- TEvPQLibTests::EvPQLibProducerCreateResponse,
- TEvPQLibTests::EvPQLibProducerCommitResponse
- >;
-
- THolder<TProducerType> wrapper = MakeHolder<TProducerType>(runtime.GetAnyNodeActorSystem(), edgeActorId, std::move(producerHolder));
-
- // checks
- auto actorId = wrapper->GetActorID();
- UNIT_ASSERT(actorId);
-
- {
- wrapper->Start(TInstant::Max());
- TAutoPtr<NActors::IEventHandle> handle;
- auto* startEvent = runtime.GrabEdgeEvent<TProducerType::TCreateResponseEvent>(handle);
- UNIT_ASSERT(startEvent != nullptr);
- UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId);
- UNIT_ASSERT_VALUES_EQUAL(startEvent->Response.Response.GetInit().GetMaxSeqNo(), 42);
- }
-
- {
- wrapper->Write(data);
- TAutoPtr<NActors::IEventHandle> handle;
- auto* writeEvent = runtime.GrabEdgeEvent<TProducerType::TCommitResponseEvent>(handle);
- UNIT_ASSERT(writeEvent != nullptr);
- UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId);
- UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.Response.GetAck().GetSeqNo(), 100);
- UNIT_ASSERT_EQUAL(writeEvent->Response.Data, data);
- UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.SeqNo, 100);
- }
-
- {
- TError err;
- err.SetDescription("trololo");
- deadPromise.SetValue(err);
-
- TAutoPtr<NActors::IEventHandle> handle;
- auto* deadEvent = runtime.GrabEdgeEvent<TProducerType::TObjectIsDeadEvent>(handle);
- UNIT_ASSERT(deadEvent != nullptr);
- UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId);
- UNIT_ASSERT_STRINGS_EQUAL(deadEvent->Response.GetDescription(), "trololo");
- }
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h
deleted file mode 100644
index 630e8a35f8..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#pragma once
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/hfunc.h>
-
-namespace NPersQueue {
-
-// Fake object to hold actor id
-class TFakeActor: public NActors::TActor<TFakeActor> {
-public:
- TFakeActor()
- : TActor<TFakeActor>(&TFakeActor::StateFunc)
- {
- }
-
-private:
- STFUNC(StateFunc) {
- switch (ev->GetTypeRewrite()) {
- CFunc(NActors::TEvents::TSystem::PoisonPill, Die);
- }
- }
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h
deleted file mode 100644
index 4ad5fb0d48..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h
+++ /dev/null
@@ -1,48 +0,0 @@
-#pragma once
-#include "responses.h"
-#include "actor_interface.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h>
-
-namespace NPersQueue {
-
-template <ui32 EventTypeId>
-class TActorLogger
- : public TActorHolder
- , public ILogger
-{
-public:
- using TLogEvent = TEvPQLibLog<EventTypeId>;
-
-public:
- explicit TActorLogger(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, int level)
- : TActorHolder(actorSystem, parentActorID)
- , Level(level)
- {
- }
-
- ~TActorLogger() override = default;
-
- void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override {
- if (!IsEnabled(level)) {
- return;
- }
- ActorSystem->Send(ParentActorID, new TLogEvent(
- msg,
- sourceId,
- sessionId,
- static_cast<NActors::NLog::EPriority>(level)
- ));
- }
-
- bool IsEnabled(int level) const override {
- return level <= Level;
- }
-
-private:
- int Level;
-};
-
-}
-
-
-
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp
deleted file mode 100644
index c663c3ac04..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp
+++ /dev/null
@@ -1,22 +0,0 @@
-#include "actor_wrappers.h"
-#include "persqueue.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h>
-
-namespace NPersQueue {
-
-TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings)
- : PQLib(new TPQLib(settings))
- , ActorSystem(actorSystem)
-{
-}
-
-TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib)
- : PQLib(std::move(pqLib))
- , ActorSystem(actorSystem)
-{
-}
-
-TPQLibActorsWrapper::~TPQLibActorsWrapper() = default;
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h
deleted file mode 100644
index 7a11416f4f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h
+++ /dev/null
@@ -1,87 +0,0 @@
-#pragma once
-#include "actor_wrappers.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <library/cpp/actors/core/actorid.h>
-
-#include <util/generic/noncopyable.h>
-#include <util/generic/ptr.h>
-
-namespace NActors {
-class TActorSystem;
-} // namespace NActors
-
-namespace NPersQueue {
-
-class TPQLibActorsWrapper : public TNonCopyable {
-public:
- explicit TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings = TPQLibSettings());
- TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib);
- ~TPQLibActorsWrapper();
-
- // Producers creation
- template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId>
- THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateProducer(
- const NActors::TActorId& parentActorID,
- const TProducerSettings& settings,
- TIntrusivePtr<ILogger> logger = nullptr,
- bool deprecated = false
- ) {
- return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>(
- ActorSystem,
- parentActorID,
- PQLib->CreateProducer(settings, std::move(logger), deprecated)
- );
- }
-
- template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId>
- THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateMultiClusterProducer(
- const NActors::TActorId& parentActorID,
- const TMultiClusterProducerSettings& settings,
- TIntrusivePtr<ILogger> logger = nullptr,
- bool deprecated = false
- ) {
- return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>(
- ActorSystem,
- parentActorID,
- PQLib->CreateMultiClusterProducer(settings, std::move(logger), deprecated)
- );
- }
-
- // Consumers creation
- template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId>
- THolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>> CreateConsumer(
- const NActors::TActorId& parentActorID,
- const TConsumerSettings& settings,
- TIntrusivePtr<ILogger> logger = nullptr,
- bool deprecated = false
- ) {
- return MakeHolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>>(
- ActorSystem,
- parentActorID,
- PQLib->CreateConsumer(settings, std::move(logger),deprecated)
- );
- }
-
- // Processors creation
- template <ui32 TOriginDataEventId>
- THolder<TProcessorActorWrapper<TOriginDataEventId>> CreateProcessor(
- const NActors::TActorId& parentActorID,
- const TProcessorSettings& settings,
- TIntrusivePtr<ILogger> logger = nullptr,
- bool deprecated = false
- ) {
- return new TProcessorActorWrapper<TOriginDataEventId>(
- ActorSystem,
- parentActorID,
- PQLib->CreateProcessor(settings, std::move(logger), deprecated)
- );
- }
-
-private:
- std::shared_ptr<TPQLib> PQLib;
- NActors::TActorSystem* ActorSystem;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h
deleted file mode 100644
index 86e732d87c..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#pragma once
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-
-#include <library/cpp/actors/core/events.h>
-#include <library/cpp/actors/core/event_local.h>
-
-namespace NPersQueue {
-
-template <class TResponseType_, ui32 EventTypeId>
-struct TPQLibResponseEvent: public NActors::TEventLocal<TPQLibResponseEvent<TResponseType_, EventTypeId>, EventTypeId> {
- using TResponseType = TResponseType_;
-
- explicit TPQLibResponseEvent(TResponseType&& response, ui64 requestId = 0)
- : Response(std::move(response))
- , RequestId(requestId)
- {
- }
-
- TResponseType Response;
- ui64 RequestId;
-};
-
-template <ui32 EventTypeId>
-struct TEvPQLibLog : NActors::TEventLocal<TEvPQLibLog<EventTypeId>, EventTypeId> {
- TEvPQLibLog(const TString& msg, const TString& sourceId, const TString& sessionId, NActors::NLog::EPriority level)
- : Message(msg)
- , SourceId(sourceId)
- , SessionId(sessionId)
- , Level(level)
- {}
-
- TString Message;
- TString SourceId;
- TString SessionId;
- NActors::NLog::EPriority Level;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp
deleted file mode 100644
index b4d8431c8a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
-namespace NPersQueue {
-Y_UNIT_TEST_SUITE(TCompatibilityTest) {
- void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response);
- }
-
- void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response);
- }
-
- void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response);
- }
-
- void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) {
- for (ui32 i = 0; i < batch.MessageSize(); ++i) {
- ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo();
- UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i);
- }
- }
-
- void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response);
- }
-
- void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response);
- }
-
- void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response);
- }
-
- TProducerSettings MakeProducerSettings(const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = true;
- producerSettings.Topic = "topic1";
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::LZOP;
- producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10);
- return producerSettings;
- }
-
- TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) {
- TConsumerSettings consumerSettings;
- consumerSettings.ClientId = "user";
- consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- consumerSettings.Topics.push_back("topic1");
- return consumerSettings;
- }
-
- Y_UNIT_TEST(ContinuesOperationsAfterPQLibDeath) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer();
-
- const size_t partitions = 1;
- testServer.AnnoyingClient->FullInit();
- testServer.AnnoyingClient->CreateTopic("rt3.dc1--topic1", partitions);
-
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, true);
- UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError());
- auto isProducerDead = producer->IsDead();
- UNIT_ASSERT(!isProducerDead.HasValue());
-
- auto consumer = PQLib->CreateConsumer(MakeConsumerSettings(testServer), logger, true);
-
- PQLib = nullptr;
-
- UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError());
- auto isConsumerDead = consumer->IsDead();
- UNIT_ASSERT(!isConsumerDead.HasValue());
-
- auto write1 = producer->Write(1, TString("blob1"));
- auto write2 = producer->Write(2, TString("blob2"));
-
- UNIT_ASSERT(!write1.GetValueSync().Response.HasError());
- UNIT_ASSERT(!write2.GetValueSync().Response.HasError());
-
- UNIT_ASSERT(!consumer->GetNextMessage().GetValueSync().Response.HasError());
-
- producer = nullptr;
- consumer = nullptr;
-
- isProducerDead.GetValueSync();
- isConsumerDead.GetValueSync();
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h
deleted file mode 100644
index 48df7b0cdd..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#pragma once
-
-#include "logger.h"
-
-#include <kikimr/yndx/api/protos/persqueue.pb.h>
-
-#include <library/cpp/tvmauth/client/facade.h>
-
-namespace NPersQueue {
-
-const TString& GetTvmPqServiceName();
-
-class ICredentialsProvider {
-public:
- virtual ~ICredentialsProvider() = default;
- virtual void FillAuthInfo(NPersQueueCommon::TCredentials* authInfo) const = 0;
-};
-
-
-class ITVMCredentialsForwarder : public ICredentialsProvider {
-public:
- virtual ~ITVMCredentialsForwarder() = default;
- virtual void SetTVMServiceTicket(const TString& ticket) = 0;
-};
-
-std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider();
-
-/*
- * Builder for direct usage of TVM. For Qloud environment use CreateTVMQloudCredentialsProvider
- */
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger = nullptr);
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName());
-
-/*
- * forward TVM service ticket with ITMVCredentialsProvider method SetTVMServiceTicket ; threadsafe
- */
-std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder();
-
-/*
- * tvmClient settings must contain LogBroker client id under an alias
- */
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName());
-
-std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1);
-std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1);
-std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token);
-
-/*
- * Provider that takes IAM ticket from metadata service.
- */
-std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger);
-
-/*
- * Provider that makes IAM ticket from JWT token.
- */
-std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder(const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger,
- const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1),
- const TDuration& requestTimeout = TDuration::Seconds(10));
-
-std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder(const TString& jwtParams, TIntrusivePtr<ILogger> logger,
- const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1),
- const TDuration& requestTimeout = TDuration::Seconds(10));
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h
deleted file mode 100644
index 975b2ccd72..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#pragma once
-
-#include "types.h"
-#include "responses.h"
-
-#include <kikimr/yndx/api/protos/persqueue.pb.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/vector.h>
-
-namespace NPersQueue {
-
-class IConsumer {
-public:
- virtual ~IConsumer() = default;
-
- // Start consumer.
- // Consumer can be used after its start will be finished.
- virtual NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0;
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TDuration timeout) noexcept {
- return Start(TInstant::Now() + timeout);
- }
-
- //read result according to settings or Lock/Release requests from server(if UseLockSession is true)
- virtual NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept = 0;
-
- //commit processed reads
- virtual void Commit(const TVector<ui64>& cookies) noexcept = 0;
-
- //will request status - response will be received in GetNextMessage with EMT_STATUS
- virtual void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { Y_UNUSED(topic); Y_UNUSED(partition); Y_UNUSED(generation); };
-
- //will be signalled in case of errors - you may use this future if you can't use GetNextMessage() //out of memory or any other reason
- virtual NThreading::TFuture<TError> IsDead() noexcept = 0;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp
deleted file mode 100644
index c29b1d8dd8..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp
+++ /dev/null
@@ -1,421 +0,0 @@
-#include "channel.h"
-#include "channel_p.h"
-#include "internals.h"
-#include "persqueue_p.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
-
-#include <util/random/random.h>
-#include <util/string/builder.h>
-
-#include <grpc++/create_channel.h>
-
-const ui32 MAX_MESSAGE_SIZE = 150 * 1024 * 1024;
-const TDuration KEEP_ALIVE_TIME = TDuration::Seconds(90);
-const TDuration KEEP_ALIVE_TIMEOUT = TDuration::Seconds(1);
-constexpr ui64 MAGIC_COOKIE_VALUE = 123456789;
-
-namespace NPersQueue {
-
-void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials) {
- context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, GetToken(credentials));
- context.AddMetadata(NYdb::YDB_DATABASE_HEADER, database);
-}
-
-TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint) {
- TServerSetting ret = server;
- ret.UseLogbrokerCDS = EClusterDiscoveryUsageMode::DontUse;
- TStringBuf endpointWithPort = endpoint;
- TStringBuf host, port;
- ui16 parsedPort = 0;
- if (endpointWithPort.TryRSplit(':', host, port) && TryFromString(port, parsedPort)) {
- ret.Address = TString(host);
- ret.Port = parsedPort;
- } else {
- ret.Address = endpoint;
- }
- return ret;
-}
-
-bool UseCDS(const TServerSetting& server) {
- switch (server.UseLogbrokerCDS) {
- case EClusterDiscoveryUsageMode::Auto:
- return server.Address == "logbroker.yandex.net"sv || server.Address == "logbroker-prestable.yandex.net"sv;
- case EClusterDiscoveryUsageMode::Use:
- return true;
- case EClusterDiscoveryUsageMode::DontUse:
- return false;
- }
-}
-
-class TChannelImpl::TGetProxyHandler : public IHandler {
-public:
- TGetProxyHandler(TChannelImplPtr ptr)
- : Ptr(ptr)
- {}
-
- void Done() override {
- Ptr->OnGetProxyDone();
- }
-
- void Destroy(const TError& reason) override {
- Ptr->Destroy(reason);
- }
-
- TString ToString() override { return "GetProxyHandler"; }
-
-protected:
- TChannelImplPtr Ptr;
-};
-
-class TChannelOverCdsBaseImpl::TCdsResponseHandler : public IHandler {
-public:
- TCdsResponseHandler(TChannelOverCdsBaseImplPtr ptr)
- : Ptr(ptr)
- {}
-
- void Done() override {
- Ptr->OnCdsRequestDone();
- }
-
- void Destroy(const TError& reason) override {
- Ptr->Destroy(reason);
- }
-
- TString ToString() override { return "CDS-ResponseHandler"; }
-
-protected:
- TChannelOverCdsBaseImplPtr Ptr;
-};
-
-TChannel::TChannel(
- const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-) {
- MakeImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy);
-}
-
-TChannel::TChannel(
- const TProducerSettings& settings, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-) {
- if (UseCDS(settings.Server))
- Impl = new TProducerChannelOverCdsImpl(settings, pqLib, std::move(logger), preferLocalProxy);
- else
- MakeImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy);
-}
-
-void TChannel::MakeImpl(
- const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider,
- TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-) {
- Impl = MakeIntrusive<TChannelOverDiscoveryImpl>(
- server, credentialsProvider, pqLib, std::move(logger), preferLocalProxy
- );
-}
-
-NThreading::TFuture<TChannelInfo> TChannel::GetChannel() {
- return Impl->GetChannel();
-}
-
-TChannel::~TChannel() {
- Impl->TryCancel();
- Impl->Wait();
-}
-
-void TChannel::Start() {
- Impl->Start();
-}
-
-TChannelImpl::TChannelImpl(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy)
- : Promise(NThreading::NewPromise<TChannelInfo>())
- , Server(server)
- , SelectedEndpoint(Server.GetFullAddressString())
- , CredentialsProvider(credentialsProvider)
- , PreferLocalProxy(preferLocalProxy)
- , Logger(std::move(logger))
- , PQLib(pqLib)
- , CQ(pqLib->GetCompletionQueue())
- , ChooseProxyFinished(0)
-{}
-
-TChannelOverDiscoveryImpl::TChannelOverDiscoveryImpl(
- const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-)
- : TChannelImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy)
-{
- CreationTimeout = PQLib->GetSettings().ChannelCreationTimeout;
-}
-
-TChannelOverCdsBaseImpl::TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-)
- : TChannelOverDiscoveryImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy)
- , CdsRequestFinished(0)
-{}
-
-void TChannelOverDiscoveryImpl::Start() {
- Channel = CreateGrpcChannel(SelectedEndpoint);
- Stub = Ydb::Discovery::V1::DiscoveryService::NewStub(Channel);
- IHandlerPtr handler = new TGetProxyHandler(this);
- Context.set_deadline(TInstant::Now() + CreationTimeout);
- Ydb::Discovery::ListEndpointsRequest request;
- request.set_database(Server.Database);
- FillMetaHeaders(Context, Server.Database, CredentialsProvider.get());
- DEBUG_LOG("Send list endpoints request: " << request, "", "");
- Rpc = Stub->AsyncListEndpoints(&Context, request, CQ.get());
- Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler)));
-}
-
-void TChannelOverCdsBaseImpl::Start() {
- Channel = CreateGrpcChannel(Server.GetFullAddressString());
- Stub = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel);
- IHandlerPtr handler = new TCdsResponseHandler(this);
- CdsContext.set_deadline(TInstant::Now() + PQLib->GetSettings().ChannelCreationTimeout);
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request = GetCdsRequest();
- FillMetaHeaders(CdsContext, Server.Database, CredentialsProvider.get());
- DEBUG_LOG("Send cds request: " << request, "", "");
- CreationStartTime = TInstant::Now();
- Rpc = Stub->AsyncDiscoverClusters(&CdsContext, request, CQ.get());
- Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler)));
-}
-
-NThreading::TFuture<TChannelInfo> TChannelImpl::GetChannel() {
- return Promise.GetFuture();
-}
-
-bool TChannelImpl::TryCancel() {
- if (AtomicSwap(&ChooseProxyFinished, 1) == 0) {
- Y_ASSERT(!Promise.HasValue());
- Promise.SetValue(TChannelInfo{nullptr, 0});
- Context.TryCancel();
- return true;
- }
- return false;
-}
-
-TChannelImpl::~TChannelImpl() = default;
-
-std::shared_ptr<grpc::Channel> TChannelImpl::CreateGrpcChannel(const TString& address) {
- grpc::ChannelArguments args;
- args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, KEEP_ALIVE_TIME.MilliSeconds());
- args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, KEEP_ALIVE_TIMEOUT.MilliSeconds());
- args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1);
- args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0);
- args.SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, KEEP_ALIVE_TIME.MilliSeconds());
-
- args.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, MAX_MESSAGE_SIZE);
- args.SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, MAX_MESSAGE_SIZE);
-
- //TODO: fill there other settings from PQLib
- DEBUG_LOG("Creating grpc channel to \"" << address << "\"", "", "");
- if (Server.UseSecureConnection)
- return grpc::CreateCustomChannel(address,
- grpc::SslCredentials(grpc::SslCredentialsOptions{Server.CaCert, "", ""}),
- args);
- else
- return grpc::CreateCustomChannel(address, grpc::InsecureChannelCredentials(), args);
-}
-
-TString TChannelImpl::GetProxyAddress(const TString& proxyName, ui32 port) {
- TStringBuilder address;
- if (proxyName.find(':') != TString::npos && proxyName.find('[') == TString::npos) { //fix for ipv6 adresses
- address << "[" << proxyName << "]";
- } else {
- address << proxyName;
- }
- if (port)
- address << ":" << port;
- else
- address << ":" << Server.Port;
- return std::move(address);
-}
-
-void TChannelImpl::OnGetProxyDone() {
- if (AtomicSwap(&ChooseProxyFinished, 1) == 0) {
- const bool ok = Status.ok();
- Y_ASSERT(!Promise.HasValue());
- if (ok) {
- ProcessGetProxyResponse();
- } else {
- if (Status.error_code() == grpc::CANCELLED) {
- INFO_LOG("Grpc request is canceled by user", "", "");
- } else {
- const auto& msg = Status.error_message();
- TString res(msg.data(), msg.length());
- ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", "");
- }
- StartFailed();
- }
- }
- SetDone();
-}
-
-void TChannelImpl::StartFailed() {
- AtomicSet(ChooseProxyFinished, 1);
- Promise.SetValue(TChannelInfo{nullptr, 0});
-}
-
-void TChannelOverDiscoveryImpl::ProcessGetProxyResponse() {
- Ydb::Discovery::ListEndpointsResult result;
- auto& reply = Response.operation().result();
- reply.UnpackTo(&result);
-
- auto has_service = [](const Ydb::Discovery::EndpointInfo& endpoint) {
- for (auto& service: endpoint.service()) {
- if (service == "pq")
- return true;
- }
- return false;
- };
- TVector<std::pair<TString, ui32>> validEps;
- for (const auto& ep : result.endpoints()) {
- if (has_service(ep) && (!Server.UseSecureConnection || ep.ssl())) {
- validEps.emplace_back(ep.address(), ep.port());
- }
- }
- if (validEps.empty()) {
- ERR_LOG("No valid endpoints to connect!", "", "");
- StartFailed();
- } else {
- auto& selected = validEps[RandomNumber<ui32>(validEps.size())];
- DEBUG_LOG("Selected endpoint {\"" << selected.first << "\", " << selected.second << "}", "", "");
- auto channel = CreateGrpcChannel(GetProxyAddress(selected.first, selected.second));
- Promise.SetValue(TChannelInfo{std::move(channel), MAGIC_COOKIE_VALUE});
- }
-}
-
-void TChannelOverCdsBaseImpl::OnCdsRequestDone() {
- DEBUG_LOG("ON CDS request done: " << Response, "", "");
-
- if (AtomicSwap(&CdsRequestFinished, 1) == 0) {
- const bool ok = Status.ok();
- if (ok) {
- if (ProcessCdsResponse()) {
- CreationTimeout = CreationTimeout - (TInstant::Now() - CreationStartTime);
- TChannelOverDiscoveryImpl::Start();
- // Don't set AllDone event, because we are about to continue node discovery
- // via TChannelOverDiscoveryImpl part of implementation.
- return;
- } else {
- StartFailed();
- }
- } else {
- if (Status.error_code() == grpc::CANCELLED) {
- INFO_LOG("Grpc request is canceled by user", "", "");
- } else {
- const auto& msg = Status.error_message();
- TString res(msg.data(), msg.length());
- ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", "");
- }
- StartFailed();
- }
- }
- SetDone();
-}
-
-bool TChannelOverCdsBaseImpl::TryCancel() {
- if (AtomicSwap(&CdsRequestFinished, 1) == 0) {
- Y_ASSERT(!Promise.HasValue());
- Promise.SetValue(TChannelInfo{nullptr, 0});
- CdsContext.TryCancel();
- return true;
- } else {
- TChannelOverDiscoveryImpl::TryCancel();
- }
- return false;
-}
-
-TProducerChannelOverCdsImpl::TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy)
- : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy)
- , Topic(settings.Topic)
- , SourceId(settings.SourceId)
- , PreferredCluster(settings.PreferredCluster)
-{
-}
-
-Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TProducerChannelOverCdsImpl::GetCdsRequest() const {
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
- auto* params = request.add_write_sessions();
- params->set_topic(Topic);
- params->set_source_id(SourceId);
- params->set_preferred_cluster_name(PreferredCluster); // Can be empty.
- return request;
-}
-
-bool TProducerChannelOverCdsImpl::ProcessCdsResponse() {
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result;
- auto& reply = Response.operation().result();
- reply.UnpackTo(&result);
- DEBUG_LOG("Process CDS result: " << result, "", "");
- for (auto& session: result.write_sessions_clusters()) {
- for (auto& cluster: session.clusters()) {
- if (cluster.available()) {
- const TServerSetting selectedServer = ApplyClusterEndpoint(Server, cluster.endpoint());
- SelectedEndpoint = GetProxyAddress(selectedServer.Address, selectedServer.Port);
- return true;
- }
- }
- }
- ERR_LOG("Could not find valid cluster in CDS response", "", "");
- return false;
-}
-
-TConsumerChannelOverCdsImpl::TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger)
- : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, true)
- , Result(std::make_shared<TResult>())
- , Topics(settings.Topics)
-{
-}
-
-Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TConsumerChannelOverCdsImpl::GetCdsRequest() const {
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
- for (const TString& topic : Topics) {
- auto* params = request.add_read_sessions();
- params->set_topic(topic);
- params->mutable_all_original(); // set all_original
- }
- return request;
-}
-
-bool TConsumerChannelOverCdsImpl::ProcessCdsResponse() {
- auto& reply = Response.operation().result();
- reply.UnpackTo(&Result->second);
- Result->first = Status;
- DEBUG_LOG("Got CDS result: " << Result->second, "", "");
- return false; // Signals future
-}
-
-void TConsumerChannelOverCdsImpl::StartFailed() {
- Result->first = Status;
- TChannelOverCdsBaseImpl::StartFailed();
-}
-
-void TChannelImpl::SetDone() {
- AllDoneEvent.Signal();
-}
-
-void TChannelImpl::Destroy(const TError& error) {
- if (AtomicSwap(&ChooseProxyFinished, 1) == 0) {
- const auto& msg = Status.error_message();
- TString res(msg.data(), msg.length());
-
- ERR_LOG("Got proxy error response " << error << " grpc error " << res, "", "");
- //TODO: add here status from Status to response, log status
-
- Y_ASSERT(!Promise.HasValue());
- Promise.SetValue(TChannelInfo{nullptr, 0});
- }
-
- SetDone();
-}
-
-void TChannelImpl::Wait() {
- AllDoneEvent.Wait();
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h
deleted file mode 100644
index 630b097675..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <deque>
-
-namespace NPersQueue {
-
-struct TChannelInfo {
- std::shared_ptr<grpc::Channel> Channel;
- ui64 ProxyCookie;
-};
-
-class TChannel;
-using TChannelPtr = TIntrusivePtr<TChannel>;
-
-class TPQLibPrivate;
-
-struct TChannelHolder {
- TChannelPtr ChannelPtr;
- NThreading::TFuture<TChannelInfo> ChannelInfo;
-};
-
-class TChannelImpl;
-using TChannelImplPtr = TIntrusivePtr<TChannelImpl>;
-
-class TChannel: public TAtomicRefCount<TChannel> {
-public:
- friend class TPQLibPrivate;
-
- NThreading::TFuture<TChannelInfo> GetChannel();
-
- ~TChannel();
-
- void Start();
-
-private:
- TChannel(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false);
- TChannel(const TProducerSettings& settings, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false);
- void MakeImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false);
- TChannelImplPtr Impl;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h
deleted file mode 100644
index 10f18abb04..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h
+++ /dev/null
@@ -1,156 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include "channel.h"
-#include "scheduler.h"
-#include "internals.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include <library/cpp/threading/future/future.h>
-#include <ydb/public/api/protos/ydb_discovery.pb.h>
-#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h>
-#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
-
-#include <deque>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-// Apply endpoint that was taken from CDS.
-TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint);
-
-class TChannelImpl : public TAtomicRefCount<TChannelImpl> {
-public:
- NThreading::TFuture<TChannelInfo> GetChannel();
-
- virtual ~TChannelImpl();
-
- virtual bool TryCancel();
-
- TChannelImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy);
-
- virtual void Start() = 0;
-
- void Wait();
-
-protected:
- friend class TPQLibPrivate;
- class TGetProxyHandler;
-
- void Destroy(const TError& error);
- virtual void StartFailed();
- void SetDone();
- void OnGetProxyDone();
- virtual void ProcessGetProxyResponse() = 0;
- std::shared_ptr<grpc::Channel> CreateGrpcChannel(const TString& address);
- TString GetProxyAddress(const TString& proxyName, ui32 port = 0);
- TString GetToken();
-
- NThreading::TPromise<TChannelInfo> Promise;
- TServerSetting Server;
- TString SelectedEndpoint;
- std::shared_ptr<ICredentialsProvider> CredentialsProvider;
- bool PreferLocalProxy;
-
- TIntrusivePtr<ILogger> Logger;
- TPQLibPrivate* PQLib;
-
-protected:
- std::shared_ptr<grpc::CompletionQueue> CQ;
- std::shared_ptr<grpc::Channel> Channel;
- grpc::ClientContext Context;
- grpc::Status Status;
-
- TAutoEvent AllDoneEvent;
-
- TAtomic ChooseProxyFinished;
-};
-
-class TChannelOverDiscoveryImpl : public TChannelImpl {
-public:
- TChannelOverDiscoveryImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy);
- void Start() override;
-
-protected:
- void ProcessGetProxyResponse() override;
- TDuration CreationTimeout;
-
-private:
- std::unique_ptr<Ydb::Discovery::V1::DiscoveryService::Stub> Stub;
- std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::Discovery::ListEndpointsResponse>> Rpc;
- Ydb::Discovery::ListEndpointsResponse Response;
-};
-
-
-class TChannelOverCdsBaseImpl : public TChannelOverDiscoveryImpl {
-public:
- TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy);
- void Start() override;
- void OnCdsRequestDone();
-
- bool TryCancel() override;
-
-private:
- virtual Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const = 0;
- virtual bool ProcessCdsResponse() = 0;
-
-protected:
- class TCdsResponseHandler;
-
-private:
- std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub;
- std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>> Rpc;
-
-protected:
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse Response;
-
-private:
- grpc::ClientContext CdsContext;
-
- TInstant CreationStartTime = TInstant::Zero();
- TAtomic CdsRequestFinished;
-};
-using TChannelOverCdsBaseImplPtr = TIntrusivePtr<TChannelOverCdsBaseImpl>;
-
-class TProducerChannelOverCdsImpl : public TChannelOverCdsBaseImpl {
-public:
- TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy);
-
-private:
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override;
- bool ProcessCdsResponse() override;
-
-private:
- TString Topic;
- TString SourceId;
- TString PreferredCluster;
-};
-
-class TConsumerChannelOverCdsImpl : public TChannelOverCdsBaseImpl {
-public:
- using TResult = std::pair<grpc::Status, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult>;
- using TResultPtr = std::shared_ptr<TResult>;
-
-public:
- TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib,
- TIntrusivePtr<ILogger> logger);
-
- TResultPtr GetResultPtr() const {
- return Result;
- }
-
-private:
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override;
- bool ProcessCdsResponse() override;
- void StartFailed() override;
-
-private:
- TResultPtr Result;
- TVector<TString> Topics;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp
deleted file mode 100644
index 14c2871a54..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp
+++ /dev/null
@@ -1,256 +0,0 @@
-
-#include "compat_producer.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
- using namespace NYdb::NPersQueue;
-
-
- static TWriteSessionSettings ConvertToWriteSessionSettings(const TProducerSettings& ps) {
- TWriteSessionSettings settings;
- settings.Path(ps.Topic)
- .MessageGroupId(ps.SourceId)
- .RetryPolicy(ps.ReconnectOnFailure ? NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(ps.ReconnectionDelay, ps.ReconnectionDelay, ps.MaxReconnectionDelay,
- ps.MaxAttempts, TDuration::Max(), 2.0, [](NYdb::EStatus){ return ERetryErrorClass::ShortRetry; })
- : NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy())
- .Codec(NYdb::NPersQueue::ECodec(ps.Codec + 1))
- .CompressionLevel(ps.Quality)
- .ValidateSeqNo(false);
- for (auto& attr : ps.ExtraAttrs) {
- settings.AppendSessionMeta(attr.first, attr.second);
- }
-
- if (ps.PreferredCluster) {
- settings.PreferredCluster(ps.PreferredCluster);
- }
- if (ps.DisableCDS || ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) {
- settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Off);
- } else if (ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::Use) {
- settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::On);
- }
-
- return settings;
- }
-
- TYdbSdkCompatibilityProducer::~TYdbSdkCompatibilityProducer() {
- WriteSession->Close(TDuration::Seconds(0));
-
- NotifyClient(NErrorCode::OK, "producer object destroyed by client");
- }
-
- TYdbSdkCompatibilityProducer::TYdbSdkCompatibilityProducer(const TProducerSettings& settings, TPersQueueClient& persQueueClient,
- std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib)
- : IProducerImpl(destroyEventRef, pqLib)
- {
- IsDeadPromise = ::NThreading::NewPromise<TError>();
- TWriteSessionSettings sessionSettings = ConvertToWriteSessionSettings(settings);
- Closed = false;
- WriteSession = persQueueClient.CreateWriteSession(sessionSettings);
- }
-
- void TYdbSdkCompatibilityProducer::SubscribeToNextEvent() {
- NextEvent = WriteSession->WaitEvent();
- std::weak_ptr<TYdbSdkCompatibilityProducer> self = shared_from_this();
- NextEvent.Subscribe([self](const auto& future) {
- auto selfShared = self.lock();
- Y_UNUSED(future);
- if (selfShared) {
- selfShared->DoProcessNextEvent();
- }
- });
- }
-
- void TYdbSdkCompatibilityProducer::Init() noexcept {
- }
-
- NThreading::TFuture<TProducerCreateResponse> TYdbSdkCompatibilityProducer::Start(TInstant deadline) noexcept {
- Y_UNUSED(deadline); // TODO: start timeout?
- SubscribeToNextEvent();
- return WriteSession->GetInitSeqNo().Apply([](const auto& future) {
- TWriteResponse response;
- if (future.HasException()) {
- response.MutableError()->SetDescription("session closed");
- response.MutableError()->SetCode(NErrorCode::ERROR);
- } else {
- response.MutableInit()->SetMaxSeqNo(future.GetValue());
- }
- TProducerCreateResponse res(std::move(response));
-
- return NThreading::MakeFuture<TProducerCreateResponse>(res);
- });
- }
-
- void TYdbSdkCompatibilityProducer::WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, ::NPersQueue::TData data) noexcept {
- TContinuationToken *contToken = nullptr;
- bool write = false;
- with_lock(Lock) {
- if (!Closed) {
- contToken = ContToken.Get();
- if(contToken == nullptr || !ToWrite.empty()) {
- ToWrite.emplace(promise, seqNo, data);
- return;
- } else { // contToken here and ToWrite is empty - can write right now
- ToAck.emplace(promise, seqNo, data);
- write = true;
- }
- }
- }
- if (write) {
- if (data.IsEncoded()) {
- WriteSession->WriteEncoded(std::move(*contToken), data.GetEncodedData(), NYdb::NPersQueue::ECodec(data.GetCodecType() + 1), data.GetOriginalSize(), seqNo); // can trigger NextEvent.Subscribe() in the same thread
- } else {
- WriteSession->Write(std::move(*contToken), data.GetSourceData(), seqNo); // can trigger NextEvent.Subscribe() in the same thread
- }
- } else {
- TWriteResponse wr;
- wr.MutableError()->SetCode(NErrorCode::ERROR);
- wr.MutableError()->SetDescription("session closed");
- TProducerCommitResponse resp(seqNo ? *seqNo : 0, data, std::move(wr));
- promise.SetValue(resp);
- }
- }
-
- void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, ::NPersQueue::TData data) noexcept {
- return WriteImpl(promise, {}, std::move(data)); // similar to TCompressingProducer
- }
-
- void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, ::NPersQueue::TData data) noexcept {
- return WriteImpl(promise, seqNo, std::move(data)); // similar to TCompressingProducer
- }
-
- void TYdbSdkCompatibilityProducer::Cancel() {
- WriteSession->Close(TDuration::Seconds(0));
- }
-
- NErrorCode::EErrorCode GetErrorCode(const NYdb::EStatus status) {
- switch(status) {
- case NYdb::EStatus::SUCCESS:
- return NErrorCode::OK;
- case NYdb::EStatus::UNAVAILABLE:
- return NErrorCode::INITIALIZING;
- case NYdb::EStatus::OVERLOADED:
- return NErrorCode::OVERLOAD;
- case NYdb::EStatus::BAD_REQUEST:
- return NErrorCode::BAD_REQUEST;
- case NYdb::EStatus::NOT_FOUND:
- case NYdb::EStatus::SCHEME_ERROR:
- return NErrorCode::UNKNOWN_TOPIC;
- case NYdb::EStatus::UNSUPPORTED:
- return NErrorCode::BAD_REQUEST;
- case NYdb::EStatus::UNAUTHORIZED:
- return NErrorCode::ACCESS_DENIED;
- default:
- return NErrorCode::ERROR;
- }
- return NErrorCode::ERROR;
- }
-
- struct TToNotify {
- NThreading::TPromise<TProducerCommitResponse> Promise;
- TProducerCommitResponse Resp;
-
- TToNotify(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerCommitResponse&& resp)
- : Promise(promise)
- , Resp(resp)
- {}
- };
-
-
- void TYdbSdkCompatibilityProducer::NotifyClient(NErrorCode::EErrorCode code, const TString& reason) {
-
- TError err;
- err.SetCode(code);
- err.SetDescription(reason);
- std::queue<TToNotify> toNotify;
- with_lock(Lock) {
- if (Closed) return;
- Closed = true;
- while(!ToAck.empty()) {
- TMsgData& item = ToAck.front();
- TWriteResponse wr;
- wr.MutableError()->CopyFrom(err);
- TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr));
- toNotify.emplace(item.Promise, std::move(resp));
- ToAck.pop();
- }
- while(!ToWrite.empty()) {
- TMsgData& item = ToWrite.front();
- TWriteResponse wr;
- wr.MutableError()->CopyFrom(err);
- TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr));
- toNotify.emplace(item.Promise, std::move(resp));
- ToWrite.pop();
- }
- }
- while(!toNotify.empty()) {
- toNotify.front().Promise.SetValue(toNotify.front().Resp);
- toNotify.pop();
- }
- IsDeadPromise.SetValue(err);
- DestroyPQLibRef();
- }
-
- void TYdbSdkCompatibilityProducer::DoProcessNextEvent() {
-
- std::queue<TToNotify> toNotify;
-
- TVector<TWriteSessionEvent::TEvent> events = WriteSession->GetEvents(false);
- //Y_VERIFY(!events.empty());
- for (auto& event : events) {
- if(std::holds_alternative<TSessionClosedEvent>(event)) {
- NotifyClient(GetErrorCode(std::get<TSessionClosedEvent>(event).GetStatus()), std::get<TSessionClosedEvent>(event).DebugString() );
- return;
- } else if(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)) {
-
- // get seqNos, signal their promises
- TWriteSessionEvent::TAcksEvent& acksEvent = std::get<TWriteSessionEvent::TAcksEvent>(event);
- std::queue<TToNotify> toNotify;
- for(TWriteSessionEvent::TWriteAck& ackEvent : acksEvent.Acks) {
- TWriteResponse writeResp;
- auto ackMsg = writeResp.MutableAck();
- ackMsg->SetSeqNo(ackEvent.SeqNo);
- ackMsg->SetAlreadyWritten(ackEvent.State == TWriteSessionEvent::TWriteAck::EEventState::EES_ALREADY_WRITTEN);
- ackMsg->SetOffset(ackEvent.Details ? ackEvent.Details->Offset : 0);
- with_lock(Lock) {
- Y_ASSERT(!ToAck.empty());
- TProducerCommitResponse resp(ackEvent.SeqNo, ToAck.front().Data, std::move(writeResp));
- toNotify.emplace(ToAck.front().Promise, std::move(resp));
- ToAck.pop();
- }
- }
- while(!toNotify.empty()) {
- toNotify.front().Promise.SetValue(toNotify.front().Resp);
- toNotify.pop();
- }
- } else if(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) {
- TWriteSessionEvent::TReadyToAcceptEvent& readyEvent = std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event);
- TContinuationToken contToken = std::move(readyEvent.ContinuationToken); // IF there is only one only movable contToken THEN this is thread safe
- TString strData;
- TMaybe<TProducerSeqNo> seqNo;
- bool write = false;
- with_lock(Lock) {
- if(ToWrite.empty()) {
- ContToken = std::move(contToken);
- } else {
- strData = ToWrite.front().Data.GetSourceData();
- seqNo = ToWrite.front().SeqNo;
- ToAck.push(ToWrite.front());
- ToWrite.pop();
- write = true;
- }
- }
- if (write) {
- WriteSession->Write(std::move(contToken), strData, seqNo);
- }
- }
- }
-
- SubscribeToNextEvent();
- }
-
- NThreading::TFuture<TError> TYdbSdkCompatibilityProducer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
- }
-
-} // namespace NYdb::NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h
deleted file mode 100644
index e5287d4694..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#pragma once
-
-#include "iproducer_p.h"
-
-#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h>
-
-#include <memory>
-#include <queue>
-
-namespace NPersQueue {
-
-
-/**
- * Compatibility producer:
- * wraps old "interface" around the new one
- */
-
-class TYdbSdkCompatibilityProducer: public IProducerImpl, public std::enable_shared_from_this<TYdbSdkCompatibilityProducer> {
-
- struct TMsgData {
- NThreading::TPromise<TProducerCommitResponse> Promise;
- TMaybe<TProducerSeqNo> SeqNo;
- TData Data;
-
- TMsgData() {};
- TMsgData(NThreading::TPromise<TProducerCommitResponse> promise, TMaybe<TProducerSeqNo> seqNo, TData data)
- : Promise(promise)
- , SeqNo(seqNo)
- , Data(data) {}
- };
-
- std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession;
- NThreading::TPromise<TError> IsDeadPromise;
- NThreading::TFuture<void> NextEvent;
-
- TMaybe<NYdb::NPersQueue::TContinuationToken> ContToken;
- std::queue<TMsgData> ToWrite;
- std::queue<TMsgData> ToAck;
- TSpinLock Lock;
- bool Closed;
-
- void DoProcessNextEvent();
- void SubscribeToNextEvent();
-
- void WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, TData data) noexcept;
-
- void NotifyClient(NErrorCode::EErrorCode code, const TString& reason);
-
-public:
- using IProducerImpl::Write;
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override;
- NThreading::TFuture<TError> IsDead() noexcept override;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override;
- void Cancel() override;
- void Init() noexcept override;
-
- ~TYdbSdkCompatibilityProducer();
-
- TYdbSdkCompatibilityProducer(const TProducerSettings& settings, NYdb::NPersQueue::TPersQueueClient& persQueueClient,
- std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib);
-
-};
-
-
-} // namespace NYdb::NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp
deleted file mode 100644
index ecb6f70944..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp
+++ /dev/null
@@ -1,155 +0,0 @@
-#include "compressing_producer.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-TCompressingProducer::TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger)
- : IProducerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Logger(std::move(logger))
- , Subproducer(std::move(subproducer))
- , DefaultCodec(defaultCodec)
- , Quality(quality)
-{
- if (defaultCodec == ECodec::DEFAULT) {
- ythrow yexception() << "Producer codec can't be ECodec::DEFAULT";
- }
-}
-
-TCompressingProducer::~TCompressingProducer() {
- DestroyQueue("Destructor called");
-}
-
-NThreading::TFuture<TProducerCreateResponse> TCompressingProducer::Start(TInstant deadline) noexcept {
- return Subproducer->Start(deadline);
-}
-
-NThreading::TFuture<TData> TCompressingProducer::Enqueue(TData data) {
- NThreading::TPromise<TData> promise = NThreading::NewPromise<TData>();
- std::weak_ptr<TCompressingProducer> self = shared_from_this();
- const void* queueTag = this;
- const ECodec defaultCodec = DefaultCodec;
- const int quality = Quality;
- auto compress = [promise, self, data, queueTag, defaultCodec, quality, pqLib = PQLib.Get()]() mutable {
- Compress(promise, std::move(data), defaultCodec, quality, queueTag, self, pqLib);
- };
- Y_VERIFY(PQLib->GetCompressionPool().AddFunc(compress));
- return promise.GetFuture();
-}
-
-void TCompressingProducer::Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality, const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) {
- if (!self.lock()) {
- return;
- }
- promise.SetValue(TData::Encode(std::move(data), defaultCodec, quality));
- SignalProcessQueue(queueTag, self, pqLib);
-}
-
-void TCompressingProducer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) {
- auto processQueue = [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->ProcessQueue();
- }
- };
-
- //can't use this->Logger here - weak_ptr::lock -> ~TCompressingProducer in compression thread ->
- //race with cancel() in main thread pool
- Y_UNUSED(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue));
-}
-
-void TCompressingProducer::ProcessQueue() {
- if (DestroyedFlag) {
- return;
- }
-
- size_t removeCount = 0;
- bool prevSignalled = true; // flag to guarantee correct order of futures signalling.
- for (TWriteRequestInfo& request : Queue) {
- if (!request.EncodedData.HasValue()) {
- // stage 1
- break;
- }
- if (!request.Future.Initialized()) {
- // stage 2
- if (request.SeqNo) {
- request.Future = Subproducer->Write(request.SeqNo, request.EncodedData.GetValue());
- } else {
- request.Future = Subproducer->Write(request.EncodedData.GetValue());
- }
- std::weak_ptr<TCompressingProducer> self = shared_from_this();
- const void* queueTag = this;
- auto signalProcess = [self, queueTag, pqLib = PQLib.Get()](const auto&) {
- SignalProcessQueue(queueTag, self, pqLib);
- };
- request.Future.Subscribe(signalProcess);
- } else {
- // stage 3
- if (prevSignalled && request.Future.HasValue()) {
- request.Promise.SetValue(request.Future.GetValue());
- ++removeCount;
- } else {
- prevSignalled = false;
- }
- }
- }
- while (removeCount--) {
- Queue.pop_front();
- }
-}
-
-void TCompressingProducer::DestroyQueue(const TString& reason) {
- if (DestroyedFlag) {
- return;
- }
-
- DestroyedFlag = true;
- for (TWriteRequestInfo& request : Queue) {
- if (request.Future.Initialized() && request.Future.HasValue()) {
- request.Promise.SetValue(request.Future.GetValue());
- } else {
- TWriteResponse resp;
- TError& err = *resp.MutableError();
- err.SetCode(NErrorCode::ERROR);
- err.SetDescription(reason);
- request.Promise.SetValue(TProducerCommitResponse(request.SeqNo, std::move(request.Data), std::move(resp)));
- }
- }
- Queue.clear();
-
- DestroyPQLibRef();
-}
-
-void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept {
- if (DestroyedFlag) {
- return;
- }
-
- Queue.emplace_back(seqNo, data, promise);
- if (data.IsEncoded()) {
- Queue.back().EncodedData = NThreading::MakeFuture<TData>(data);
- ProcessQueue();
- } else if (DefaultCodec == ECodec::RAW) {
- Queue.back().EncodedData = NThreading::MakeFuture<TData>(TData::MakeRawIfNotEncoded(data));
- ProcessQueue();
- } else {
- Queue.back().EncodedData = Enqueue(std::move(data));
- }
-}
-
-void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept {
- TCompressingProducer::Write(promise, 0, std::move(data));
-}
-
-NThreading::TFuture<TError> TCompressingProducer::IsDead() noexcept {
- return Subproducer->IsDead();
-}
-
-NThreading::TFuture<void> TCompressingProducer::Destroyed() noexcept {
- return WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subproducer->Destroyed());
-}
-
-void TCompressingProducer::Cancel() {
- DestroyQueue(GetCancelReason());
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h
deleted file mode 100644
index 6ed9e89bed..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h
+++ /dev/null
@@ -1,64 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include "internals.h"
-#include "iproducer_p.h"
-
-#include <deque>
-#include <memory>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class TCompressingProducer: public IProducerImpl, public std::enable_shared_from_this<TCompressingProducer> {
-public:
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override;
- using IProducerImpl::Write;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override;
- NThreading::TFuture<TError> IsDead() noexcept override;
- NThreading::TFuture<void> Destroyed() noexcept override;
-
- ~TCompressingProducer();
-
- TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger);
-
- void Cancel() override;
-
-private:
- struct TWriteRequestInfo {
- TWriteRequestInfo(TProducerSeqNo seqNo, TData data, const NThreading::TPromise<TProducerCommitResponse>& promise)
- : SeqNo(seqNo)
- , Data(std::move(data))
- , Promise(promise)
- {
- }
-
- TProducerSeqNo SeqNo;
- TData Data;
- NThreading::TFuture<TData> EncodedData;
- NThreading::TPromise<TProducerCommitResponse> Promise;
- NThreading::TFuture<TProducerCommitResponse> Future;
- };
-
- void ProcessQueue();
- NThreading::TFuture<TData> Enqueue(TData data);
- // these functions should be static and not lock self in compression thread pool for proper futures ordering
- static void Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality,
- const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib);
- static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib);
- void DestroyQueue(const TString& reason);
-
-protected:
- TIntrusivePtr<ILogger> Logger;
- std::shared_ptr<IProducerImpl> Subproducer;
- ECodec DefaultCodec;
- int Quality;
- std::deque<TWriteRequestInfo> Queue;
- bool DestroyedFlag = false;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp
deleted file mode 100644
index 4d9b30f69a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp
+++ /dev/null
@@ -1,243 +0,0 @@
-#include "compressing_producer.h"
-#include "local_caller.h"
-#include "persqueue_p.h"
-
-#include <library/cpp/testing/gmock_in_unittest/gmock.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/thread.h>
-
-using namespace testing;
-
-namespace NPersQueue {
-
-class TMockProducer: public IProducerImpl {
-public:
- TMockProducer()
- : IProducerImpl(nullptr, nullptr)
- {
- }
-
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override {
- return NThreading::MakeFuture<TProducerCreateResponse>(MockStart());
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override {
- return NThreading::MakeFuture<TProducerCommitResponse>(MockWriteWithSeqNo(seqNo, data));
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override {
- return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data));
- }
-
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override {
- promise.SetValue(MockWriteWithSeqNo(seqNo, data));
- }
-
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override {
- promise.SetValue(MockWrite(data));
- }
-
- NThreading::TFuture<TError> IsDead() noexcept override {
- return MockIsDead();
- }
-
- void Cancel() override {
- }
-
- MOCK_METHOD(TProducerCreateResponse, MockStart, (), ());
- MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ());
- MOCK_METHOD(TProducerCommitResponse, MockWriteWithSeqNo, (TProducerSeqNo, TData), ());
- MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ());
-};
-
-template <class TMock = TMockProducer>
-struct TCompressingProducerBootstrap {
- ~TCompressingProducerBootstrap() {
- Lib->CancelObjectsAndWait();
- }
-
- void Create(ECodec codec = ECodec::GZIP) {
- Lib = new TPQLibPrivate(Settings);
- MockProducer = std::make_shared<TMock>();
- CompressingProducer = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(MockProducer, codec, 3, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger);
- Lib->AddToDestroySet(CompressingProducer);
- }
-
- void MakeOKStartResponse() {
- TWriteResponse wresp;
- wresp.MutableInit()->SetTopic("trololo");
- TProducerCreateResponse resp(std::move(wresp));
- EXPECT_CALL(*MockProducer, MockStart())
- .WillOnce(Return(resp));
- }
-
- void ExpectIsDeadCall() {
- EXPECT_CALL(*MockProducer, MockIsDead())
- .WillOnce(Return(DeadPromise.GetFuture()));
- }
-
- void Start() {
- MakeOKStartResponse();
- UNIT_ASSERT_STRINGS_EQUAL(CompressingProducer->Start().GetValueSync().Response.GetInit().GetTopic(), "trololo");
- }
-
- TPQLibSettings Settings;
- TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG);
- TIntrusivePtr<TPQLibPrivate> Lib;
- std::shared_ptr<TMock> MockProducer;
- std::shared_ptr<IProducerImpl> CompressingProducer;
- NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>();
-};
-
-Y_UNIT_TEST_SUITE(TCompressingProducerTest) {
- Y_UNIT_TEST(PassesStartAndDead) {
- TCompressingProducerBootstrap<> bootstrap;
- bootstrap.Create();
- bootstrap.ExpectIsDeadCall();
-
- TWriteResponse wresp;
- wresp.MutableError()->SetDescription("trololo");
- TProducerCreateResponse resp(std::move(wresp));
- EXPECT_CALL(*bootstrap.MockProducer, MockStart())
- .WillOnce(Return(resp));
-
- auto startFuture = bootstrap.CompressingProducer->Start();
- UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "trololo");
-
- auto isDeadFuture = bootstrap.CompressingProducer->IsDead();
- UNIT_ASSERT(isDeadFuture.Initialized());
- UNIT_ASSERT(!isDeadFuture.HasValue());
-
- TError err;
- err.SetDescription("42");
- bootstrap.DeadPromise.SetValue(err);
-
- UNIT_ASSERT_STRINGS_EQUAL(isDeadFuture.GetValueSync().GetDescription(), "42");
- }
-
- Y_UNIT_TEST(CallsWritesInProperOrder) {
- TCompressingProducerBootstrap<> bootstrap;
- bootstrap.Create();
- bootstrap.Start();
-
- TData data1 = TData("data1");
- TData expectedData1 = TData::Encode(data1, ECodec::GZIP, 3);
-
- TData data100 = TData::Raw("data100");
- TData expectedData100 = data100;
-
- TData data101 = TData("data101", ECodec::LZOP);
- TData expectedData101 = TData::Encode(data101, ECodec::GZIP, 3);
-
- TWriteResponse wresp;
- wresp.MutableAck();
-
- InSequence seq;
- EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(1, expectedData1))
- .WillOnce(Return(TProducerCommitResponse(1, expectedData1, TWriteResponse(wresp))));
-
- EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(100, expectedData100))
- .WillOnce(Return(TProducerCommitResponse(100, expectedData100, TWriteResponse(wresp))));
-
- EXPECT_CALL(*bootstrap.MockProducer, MockWrite(expectedData101))
- .WillOnce(Return(TProducerCommitResponse(101, expectedData101, TWriteResponse(wresp))));
-
- auto write1 = bootstrap.CompressingProducer->Write(1, data1);
- auto write100 = bootstrap.CompressingProducer->Write(100, data100);
- auto write101 = bootstrap.CompressingProducer->Write(data101);
-
- write1.GetValueSync();
- write100.GetValueSync();
- write101.GetValueSync();
- }
-
- class TMockProducerWithReorder: public TMockProducer {
- public:
- using TMockProducer::TMockProducer;
- using TMockProducer::Write;
-
- NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override {
- return PromiseWriteWithSeqNo(seqNo, data);
- }
-
- MOCK_METHOD(NThreading::TFuture<TProducerCommitResponse>, PromiseWriteWithSeqNo, (TProducerSeqNo, TData), ());
- };
-
- void SignalsAllFuturesInProperOrderImpl(bool death) {
- size_t lastWritten = 0;
-
- TCompressingProducerBootstrap<TMockProducerWithReorder> bootstrap;
- bootstrap.Create();
- bootstrap.Start();
-
- const size_t count = 200;
- NThreading::TFuture<TProducerCommitResponse> responses[count];
- TData datas[count];
-
- TWriteResponse wresp;
- wresp.MutableAck();
- EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(_, _))
- .Times(AtLeast(0))
- .WillRepeatedly(Return(NThreading::MakeFuture<TProducerCommitResponse>(TProducerCommitResponse(1, TData("data"), TWriteResponse(wresp)))));
-
- NThreading::TPromise<TProducerCommitResponse> promise = NThreading::NewPromise<TProducerCommitResponse>();
-
- if (death) {
- EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _))
- .Times(AtLeast(0))
- .WillRepeatedly(Return(promise.GetFuture()));
- } else {
- EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _))
- .Times(1)
- .WillRepeatedly(Return(promise.GetFuture()));
- }
-
- TString bigString(10000, 'A');
- for (size_t i = 0; i < count; ++i) {
- TData data(TStringBuilder() << bigString << i, ECodec::LZOP);
- datas[i] = data;
- }
-
- auto testThreadId = TThread::CurrentThreadId(), prevThreadId = 0ul;
- for (size_t i = 0; i < count; ++i) {
- size_t seqNo = i + 1;
- responses[i] = bootstrap.CompressingProducer->Write(i + 1, datas[i]);
- auto onSignal = [&lastWritten, &prevThreadId, seqNo, testThreadId](const auto&) {
- // proper thread
- auto curId = TThread::CurrentThreadId();
- if (prevThreadId != 0) {
- UNIT_ASSERT_C(curId == prevThreadId || curId == testThreadId, // could be executed in unittest thread if future ia already signalled or in object thread
- "prevThreadId: " << Hex(prevThreadId) << ", curId: " << Hex(curId) << ", testThreadId: " << Hex(testThreadId));
- } else if (curId != testThreadId) {
- prevThreadId = curId;
- }
-
- // proper order
- UNIT_ASSERT_VALUES_EQUAL(seqNo, lastWritten + 1);
- lastWritten = seqNo;
- };
- responses[i].Subscribe(onSignal);
- }
-
- if (death) {
- bootstrap.CompressingProducer = nullptr;
- }
-
- responses[45].GetValueSync();
- promise.SetValue(TProducerCommitResponse(49, TData("data"), TWriteResponse(wresp)));
- responses[count - 1].GetValueSync();
- for (size_t i = count - 1; i > 0; --i) {
- UNIT_ASSERT(responses[i - 1].HasValue());
- }
- }
-
- Y_UNIT_TEST(SignalsAllFuturesInProperOrderDuringDeath) {
- SignalsAllFuturesInProperOrderImpl(true);
- }
-
- Y_UNIT_TEST(SignalsAllFuturesInProperOrder) {
- SignalsAllFuturesInProperOrderImpl(false);
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp
deleted file mode 100644
index da5536ab6f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp
+++ /dev/null
@@ -1,612 +0,0 @@
-#include "consumer.h"
-#include "channel.h"
-#include "persqueue_p.h"
-
-#include <kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <util/generic/strbuf.h>
-#include <util/string/cast.h>
-#include <util/string/printf.h>
-#include <util/string/vector.h>
-#include <util/string/builder.h>
-
-#include <grpc++/create_channel.h>
-
-namespace NPersQueue {
-
-class TConsumerDestroyHandler : public IHandler {
-public:
- TConsumerDestroyHandler(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : Ptr(std::move(ptr))
- , RpcStuff(std::move(rpcStuff))
- , QueueTag(queueTag)
- , PQLib(pqLib)
- {}
-
- void Destroy(const TError& reason) override {
- auto consumer = Ptr;
- auto handler = [consumer, reason] {
- auto consumerShared = consumer.lock();
- if (consumerShared) {
- consumerShared->Destroy(reason);
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "DestroyHandler"; }
-
-protected:
- std::weak_ptr<TConsumer> Ptr;
- TIntrusivePtr<TConsumer::TRpcStuff> RpcStuff; // must simply live
- const void* QueueTag;
- TPQLibPrivate* PQLib;
-};
-
-class TConsumerStreamCreated : public TConsumerDestroyHandler {
-public:
- TConsumerStreamCreated(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto consumer = Ptr;
- auto handler = [consumer] {
- auto consumerShared = consumer.lock();
- if (consumerShared) {
- consumerShared->OnStreamCreated();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "StreamCreated"; }
-};
-
-class TConsumerWriteDone : public TConsumerDestroyHandler {
-public:
- TConsumerWriteDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto consumer = Ptr;
- auto handler = [consumer] {
- auto consumerShared = consumer.lock();
- if (consumerShared) {
- consumerShared->OnWriteDone();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "WriteDone"; }
-};
-
-class TConsumerReadDone : public TConsumerDestroyHandler {
-public:
- TConsumerReadDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto consumer = Ptr;
- auto handler = [consumer] {
- auto consumerShared = consumer.lock();
- if (consumerShared) {
- consumerShared->OnReadDone();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "ReadDone"; }
-};
-
-void TConsumer::Destroy() noexcept {
- IsDestroying = true;
- RpcStuff->Context.TryCancel();
- ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation
- Destroy("Destructor called");
-}
-
-TConsumer::TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq,
- NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept
- : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib))
- , RpcStuff(new TRpcStuff())
- , Settings(settings)
- , StartPromise(promise)
- , IsDeadPromise(NThreading::NewPromise<TError>())
- , UncommittedCount(0)
- , UncommittedSize(0)
- , MemoryUsage(0)
- , ReadsOrdered(0)
- , EstimateReadSize(settings.MaxSize)
- , WriteInflight(true)
- , ProxyCookie(0)
- , Logger(std::move(logger))
- , IsDestroyed(false)
- , IsDestroying(false)
-{
- RpcStuff->CQ = std::move(cq);
-}
-
-void TConsumer::Init()
-{
- std::weak_ptr<TConsumer> self(shared_from_this());
- StreamCreatedHandler.Reset(new TConsumerStreamCreated(self, RpcStuff, this, PQLib.Get()));
- ReadDoneHandler.Reset(new TConsumerReadDone(self, RpcStuff, this, PQLib.Get()));
- WriteDoneHandler.Reset(new TConsumerWriteDone(self, RpcStuff, this, PQLib.Get()));
-}
-
-void TConsumer::DoStart(TInstant deadline)
-{
- if (IsDestroyed)
- return;
-
- if (deadline != TInstant::Max()) {
- std::weak_ptr<TConsumer> self = shared_from_this();
- auto onStartTimeout = [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnStartTimeout();
- }
- };
- StartDeadlineCallback =
- PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout);
- }
-
- FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get());
- RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel);
- RpcStuff->Stream = RpcStuff->Stub->AsyncReadSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler));
-}
-
-NThreading::TFuture<TConsumerCreateResponse> TConsumer::Start(TInstant deadline) noexcept
-{
- NThreading::TFuture<TConsumerCreateResponse> ret = StartPromise.GetFuture();
- if (ChannelHolder.ChannelInfo.Initialized()) {
- std::weak_ptr<TConsumer> self = shared_from_this();
- PQLib->Subscribe(ChannelHolder.ChannelInfo,
- this,
- [self, deadline](const auto& f) {
- auto sharedSelf = self.lock();
- if (sharedSelf) {
- sharedSelf->SetChannel(f.GetValue());
- sharedSelf->DoStart(deadline);
- }
- });
- } else {
- if (RpcStuff->Channel && ProxyCookie) {
- DoStart(deadline);
- } else {
- Destroy("No channel");
- }
- }
- return ret;
-}
-
-void TConsumer::SetChannel(const TChannelHolder& channel) noexcept {
- if (IsDestroyed)
- return;
-
- ChannelHolder = channel;
-}
-
-void TConsumer::SetChannel(const TChannelInfo& channel) noexcept {
- if (!channel.Channel || channel.ProxyCookie == 0) {
- Destroy("Channel creation error");
- return;
- }
-
- if (IsDestroyed)
- return;
- Y_VERIFY(!RpcStuff->Channel);
- RpcStuff->Channel = channel.Channel;
- ProxyCookie = channel.ProxyCookie;
-}
-
-
-TConsumer::~TConsumer() noexcept {
- Destroy();
-}
-
-
-/************************************************read*****************************************************/
-
-void TConsumer::OrderRead() noexcept {
- if (IsDeadPromise.HasValue()) {
- return;
- }
-
- if (StartPromise.Initialized()) {
- return;
- }
-
- if (MemoryUsage >= Settings.MaxMemoryUsage || ReadsOrdered >= Settings.MaxInflyRequests) {
- return;
- }
- if (Settings.MaxUncommittedCount > 0 && UncommittedCount >= Settings.MaxUncommittedCount) {
- return;
- }
- if (Settings.MaxUncommittedSize > 0 && UncommittedSize >= Settings.MaxUncommittedSize) {
- return;
- }
-
-
- if (IsDeadPromise.HasValue()) {
- return;
- } else {
- while (ReadsOrdered < Settings.MaxInflyRequests && MemoryUsage + ((ui64)ReadsOrdered) * EstimateReadSize <= Settings.MaxMemoryUsage) {
- TReadRequest request;
- Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials());
- request.MutableRead()->SetMaxCount(Settings.MaxCount);
- request.MutableRead()->SetMaxSize(Settings.MaxSize);
- request.MutableRead()->SetPartitionsAtOnce(Settings.PartsAtOnce);
- request.MutableRead()->SetMaxTimeLagMs(Settings.MaxTimeLagMs);
- request.MutableRead()->SetReadTimestampMs(Settings.ReadTimestampMs);
- Requests.emplace_back(std::move(request));
- ++ReadsOrdered;
- DEBUG_LOG("read ordered memusage " << MemoryUsage << " infly " << ReadsOrdered << " maxmemusage "
- << Settings.MaxMemoryUsage, "", SessionId);
- }
-
- DoWrite();
- }
-}
-
-void TConsumer::OnWriteDone() {
- WriteInflight = false;
- DoWrite();
-}
-
-void TConsumer::DoWrite() {
- if (IsDestroyed) {
- return;
- }
-
- if (WriteInflight || Requests.empty())
- return;
-
- WriteInflight = true;
- TReadRequest req = Requests.front();
- Requests.pop_front();
- DEBUG_LOG("sending request " << req.GetRequestCase(), "", SessionId);
- RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler));
-}
-
-
-void TConsumer::OnStreamCreated() {
- if (IsDestroyed) {
- return;
- }
-
- TReadRequest req;
- Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials());
-
- TReadRequest::TInit* const init = req.MutableInit();
- init->SetClientId(Settings.ClientId);
- for (const auto& t : Settings.Topics) {
- init->AddTopics(t);
- }
- init->SetReadOnlyLocal(!Settings.ReadMirroredPartitions);
- init->SetClientsideLocksAllowed(Settings.UseLockSession);
- init->SetProxyCookie(ProxyCookie);
- init->SetProtocolVersion(TReadRequest::ReadParamsInInit);
- init->SetBalancePartitionRightNow(Settings.BalanceRightNow);
- init->SetCommitsDisabled(Settings.CommitsDisabled);
-
- for (auto g : Settings.PartitionGroups) {
- init->AddPartitionGroups(g);
- }
-
- // Read settings
- init->SetMaxReadMessagesCount(Settings.MaxCount);
- init->SetMaxReadSize(Settings.MaxSize);
- init->SetMaxReadPartitionsCount(Settings.PartsAtOnce);
- init->SetMaxTimeLagMs(Settings.MaxTimeLagMs);
- init->SetReadTimestampMs(Settings.ReadTimestampMs);
-
- WriteInflight = true;
- RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler));
-
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-}
-
-void TConsumer::OnReadDone() {
- if (RpcStuff->Response.HasError()) {
- Destroy(RpcStuff->Response.GetError());
- return;
- }
-
- if (IsDeadPromise.HasValue())
- return;
- if (StartPromise.Initialized()) { //init response
- NThreading::TPromise<TConsumerCreateResponse> tmp;
- tmp.Swap(StartPromise);
-
- Y_VERIFY(RpcStuff->Response.HasInit());
- auto res(std::move(RpcStuff->Response));
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-
- SessionId = res.GetInit().GetSessionId();
- DEBUG_LOG("read stream created", "", SessionId);
-
- tmp.SetValue(TConsumerCreateResponse{std::move(res)});
-
- OrderRead();
-
- return;
- }
- if (RpcStuff->Response.HasBatchedData()) {
- ConvertToOldBatch(RpcStuff->Response); // LOGBROKER-3173
- }
- if (RpcStuff->Response.HasData()) { //read response
- NThreading::TPromise<TConsumerMessage> p(NThreading::NewPromise<TConsumerMessage>());
- MessageResponses.push_back(p.GetFuture());
- const ui32 sz = RpcStuff->Response.ByteSize();
- MemoryUsage += sz;
- Y_VERIFY(ReadsOrdered);
- --ReadsOrdered;
-
- ui32 cnt = 0;
- for (ui32 i = 0; i < RpcStuff->Response.GetData().MessageBatchSize(); ++i) {
- cnt += RpcStuff->Response.GetData().GetMessageBatch(i).MessageSize();
- }
- if (!Settings.CommitsDisabled) {
- ReadInfo.push_back({RpcStuff->Response.GetData().GetCookie(), {cnt, sz}});
- UncommittedSize += sz;
- UncommittedCount += cnt;
- }
-
- DEBUG_LOG("read done memusage " << MemoryUsage << " infly " << ReadsOrdered << " size " << sz
- << " ucs " << UncommittedSize << " ucc " << UncommittedCount, "", SessionId);
-
- auto res(std::move(RpcStuff->Response));
-
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-
- OrderRead();
-
- Y_VERIFY(MemoryUsage >= sz);
- EstimateReadSize = res.ByteSize();
- OrderRead();
-
- p.SetValue(TConsumerMessage{std::move(res)});
- } else if (RpcStuff->Response.HasLock() || RpcStuff->Response.HasRelease() || RpcStuff->Response.HasCommit() || RpcStuff->Response.HasPartitionStatus()) { //lock/release/commit response
- if (!RpcStuff->Response.HasLock()) {
- INFO_LOG("got from server " << RpcStuff->Response, "", SessionId);
- if (RpcStuff->Response.HasCommit() && !Settings.CommitsDisabled) {
- for (ui32 i = 0; i < RpcStuff->Response.GetCommit().CookieSize(); ++i) {
- ui64 cookie = RpcStuff->Response.GetCommit().GetCookie(i);
- Y_VERIFY(!ReadInfo.empty() && ReadInfo.front().first == cookie);
- Y_VERIFY(UncommittedCount >= ReadInfo.front().second.first);
- Y_VERIFY(UncommittedSize >= ReadInfo.front().second.second);
- UncommittedCount -= ReadInfo.front().second.first;
- UncommittedSize -= ReadInfo.front().second.second;
- ReadInfo.pop_front();
- }
- }
- MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response))));
- OrderRead();
- } else {
- const auto& p = RpcStuff->Response.GetLock();
- NThreading::TPromise<TLockInfo> promise(NThreading::NewPromise<TLockInfo>());
- //TODO: add subscribe
- auto f = promise.GetFuture();
- std::weak_ptr<TConsumer> ptr = shared_from_this();
-
- //will take ptr for as long as promises are not set
- PQLib->Subscribe(f,
- this,
- [ptr, p](const auto& f) {
- auto consumer = ptr.lock();
- if (consumer) {
- consumer->Lock(p.GetTopic(), p.GetPartition(), p.GetGeneration(), f.GetValue().ReadOffset, f.GetValue().CommitOffset,
- f.GetValue().VerifyReadOffset);
- }
- });
-
- INFO_LOG("got LOCK from server " << p, "", SessionId);
- MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response), std::move(promise))));
- }
-
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
- } else {
- Y_FAIL("unsupported response %s", RpcStuff->Response.DebugString().c_str());
- }
-
- ProcessResponses();
-}
-
-
-/******************************************commit****************************************************/
-void TConsumer::Commit(const TVector<ui64>& cookies) noexcept {
- auto sortedCookies = cookies;
- std::sort(sortedCookies.begin(), sortedCookies.end());
-
- TReadRequest request;
- for (const auto& cookie : sortedCookies) {
- request.MutableCommit()->AddCookie(cookie);
- }
-
- DEBUG_LOG("sending COMMIT to server " << request.GetCommit(), "", SessionId);
- if (IsDestroyed || StartPromise.Initialized()) {
- ERR_LOG("trying to commit " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId);
- return;
- }
- Requests.emplace_back(std::move(request));
- DoWrite();
-}
-
-
-void TConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- TReadRequest request;
- auto status = request.MutableStatus();
- status->SetTopic(topic);
- status->SetPartition(partition);
- status->SetGeneration(generation);
-
- DEBUG_LOG("sending GET_STATUS to server " << request.GetStatus(), "", SessionId);
- if (IsDestroyed || StartPromise.Initialized()) {
- ERR_LOG("trying to get status " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId);
- return;
- }
- Requests.emplace_back(std::move(request));
- DoWrite();
-}
-
-void TConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept {
- if (IsDeadPromise.HasValue()) {
- TReadResponse res;
- res.MutableError()->SetDescription("consumer is dead");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- promise.SetValue(TConsumerMessage{std::move(res)});
- return;
- }
-
- if (StartPromise.Initialized()) {
- TReadResponse res;
- res.MutableError()->SetDescription("consumer is not ready");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- promise.SetValue(TConsumerMessage{std::move(res)});
- return;
- }
-
- MessagePromises.push_back(promise);
- ProcessResponses(); //if alredy have an answer
-}
-
-
-void TConsumer::Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset,
- const ui64 commitOffset, const bool verifyReadOffset) noexcept {
- Y_VERIFY(Settings.UseLockSession);
-
- TReadRequest request;
- request.MutableStartRead()->SetTopic(topic);
- request.MutableStartRead()->SetPartition(partition);
- request.MutableStartRead()->SetReadOffset(readOffset);
- request.MutableStartRead()->SetCommitOffset(commitOffset);
- request.MutableStartRead()->SetVerifyReadOffset(verifyReadOffset);
- request.MutableStartRead()->SetGeneration(generation);
- INFO_LOG("sending START_READ to server " << request.GetStartRead(), "", SessionId);
-
- Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials());
-
- if (IsDestroyed)
- return;
- Requests.emplace_back(std::move(request));
- DoWrite();
-}
-
-
-void TConsumer::ProcessResponses() {
- while(true) {
- if (MessagePromises.empty() || MessageResponses.empty() || !MessageResponses.front().HasValue())
- break;
- auto p(std::move(MessagePromises.front()));
- MessagePromises.pop_front();
- auto res(MessageResponses.front().ExtractValue());
- MessageResponses.pop_front();
- if (res.Type == EMT_DATA) {
- ui32 sz = res.Response.ByteSize();
- Y_VERIFY(MemoryUsage >= sz);
- MemoryUsage -= sz;
- }
-
- if (res.Type == EMT_DATA) {
- OrderRead();
- }
-
- p.SetValue(TConsumerMessage{std::move(res)});
- }
-}
-
-/***********************************************************************************/
-
-NThreading::TFuture<TError> TConsumer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TConsumer::Destroy(const TString& description) {
- TError error;
- error.SetDescription(description);
- error.SetCode(NErrorCode::ERROR);
- Destroy(error);
-}
-
-void TConsumer::Destroy(const TError& error) {
- if (!IsDestroying) {
- INFO_LOG("error: " << error, "", SessionId);
- }
-
- if (IsDestroyed)
- return;
- IsDestroyed = true;
-
- IsDeadPromise.SetValue(error);
-
- MessageResponses.clear();
-
- NThreading::TFuture<TChannelInfo> tmp;
- tmp.Swap(ChannelHolder.ChannelInfo);
- ChannelHolder.ChannelPtr = nullptr;
-
- Error = error;
- if (StartPromise.Initialized()) {
- NThreading::TPromise<TConsumerCreateResponse> tmp;
- tmp.Swap(StartPromise);
- TReadResponse res;
- res.MutableError()->CopyFrom(Error);
- tmp.SetValue(TConsumerCreateResponse{std::move(res)});
- }
-
- while (!MessagePromises.empty()) {
- auto p = MessagePromises.front();
- MessagePromises.pop_front();
- TReadResponse res;
- res.MutableError()->CopyFrom(Error);
- p.SetValue(TConsumerMessage{std::move(res)});
- }
-
- StreamCreatedHandler.Reset();
- ReadDoneHandler.Reset();
- WriteDoneHandler.Reset();
-
- DestroyPQLibRef();
-}
-
-void TConsumer::OnStartTimeout() {
- if (IsDestroyed) {
- return;
- }
-
- StartDeadlineCallback = nullptr;
- if (!StartPromise.Initialized()) {
- // already replied with successful start
- return;
- }
-
- TError error;
- error.SetDescription("Start timeout");
- error.SetCode(NErrorCode::CREATE_TIMEOUT);
- Destroy(error);
-}
-
-void TConsumer::Cancel() {
- IsDestroying = true;
- RpcStuff->Context.TryCancel();
- ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation
-
- Destroy(GetCancelReason());
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h
deleted file mode 100644
index f2f6f81e71..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h
+++ /dev/null
@@ -1,121 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include "channel.h"
-#include "scheduler.h"
-#include "iconsumer_p.h"
-#include "internals.h"
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <deque>
-#include <memory>
-
-namespace NPersQueue {
-
-class TConsumer: public IConsumerImpl, public std::enable_shared_from_this<TConsumer> {
-public:
- friend class TPQLibPrivate;
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override;
-
- void Commit(const TVector<ui64>& cookies) noexcept override;
-
- using IConsumerImpl::GetNextMessage;
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override;
-
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override;
-
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq,
- NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept;
-
- void Init() override;
-
- ~TConsumer() noexcept;
-
- void Destroy() noexcept;
- void SetChannel(const TChannelHolder& channel) noexcept;
- void SetChannel(const TChannelInfo& channel) noexcept;
-
- void Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset, const ui64 commitOffset, const bool verifyReadOffset) noexcept;
- void OrderRead() noexcept;
-
- void Cancel() override;
-
-public:
- using TStream = grpc::ClientAsyncReaderWriterInterface<TReadRequest, TReadResponse>;
-
- // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue
- struct TRpcStuff: public TAtomicRefCount<TRpcStuff> {
- TReadResponse Response;
- std::shared_ptr<grpc::CompletionQueue> CQ;
- std::shared_ptr<grpc::Channel> Channel;
- std::unique_ptr<PersQueueService::Stub> Stub;
- grpc::ClientContext Context;
- std::unique_ptr<TStream> Stream;
- };
-
-protected:
- friend class TConsumerStreamCreated;
- friend class TConsumerReadDone;
- friend class TConsumerWriteDone;
- friend class TConsumerDestroyHandler;
-
- IHandlerPtr StreamCreatedHandler;
- IHandlerPtr ReadDoneHandler;
- IHandlerPtr WriteDoneHandler;
-
- void ProcessResponses();
- void Destroy(const TError& reason);
- void Destroy(const TString& description); // the same but with Code=ERROR
- void OnStreamCreated();
- void OnReadDone();
- void OnWriteDone();
-
- void DoWrite();
- void DoStart(TInstant deadline);
- void OnStartTimeout();
-
- TIntrusivePtr<TRpcStuff> RpcStuff;
-
- TChannelHolder ChannelHolder;
- TConsumerSettings Settings;
-
- TString SessionId;
-
- NThreading::TPromise<TConsumerCreateResponse> StartPromise;
- NThreading::TPromise<TError> IsDeadPromise;
- std::deque<NThreading::TPromise<TConsumerMessage>> MessagePromises;
- std::deque<NThreading::TFuture<TConsumerMessage>> MessageResponses;
-
- std::deque<TReadRequest> Requests;
-
-
- std::deque<std::pair<ui64, std::pair<ui32, ui64>>> ReadInfo; //cookie -> (count, size)
- ui32 UncommittedCount = 0;
- ui64 UncommittedSize = 0;
-
- ui64 MemoryUsage = 0;
- ui32 ReadsOrdered = 0;
-
- ui64 EstimateReadSize = 0;
-
- bool WriteInflight;
-
- ui64 ProxyCookie = 0;
-
- TIntrusivePtr<ILogger> Logger;
-
- TError Error;
-
- bool IsDestroyed;
- bool IsDestroying;
-
- TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp
deleted file mode 100644
index dc09bd2ef1..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NPersQueue {
-Y_UNIT_TEST_SUITE(TConsumerTest) {
- Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) {
- // Test that consumer doesn't hang on till shutdown
- TPQLib lib;
- TConsumerSettings settings;
- settings.Server = TServerSetting{"localhost"};
- settings.Topics.push_back("topic");
- settings.ClientId = "client";
- lib.CreateConsumer(settings, {}, false);
- }
-
- Y_UNIT_TEST(StartTimeout) {
- if (GrpcV1EnabledByDefault()) {
- return;
- }
-
- TTestServer testServer;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic", 1);
- testServer.WaitInit("topic");
- // Test that consumer doesn't hang on till shutdown
- TConsumerSettings settings;
- settings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- settings.Topics.push_back("topic");
- settings.ClientId = "client";
- auto consumer = testServer.PQLib->CreateConsumer(settings, {}, false);
- auto startResult = consumer->Start(TInstant::Now());
- UNIT_ASSERT_EQUAL_C(startResult.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startResult.GetValueSync().Response);
-
- DestroyAndWait(consumer);
- }
-
- TProducerSettings MakeProducerSettings(const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = false;
- producerSettings.Topic = "topic1";
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::LZOP;
- return producerSettings;
- }
-
- TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) {
- TConsumerSettings settings;
- settings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- settings.Topics.emplace_back("topic1");
- settings.ClientId = "user";
- return settings;
- }
-
- Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
-
- const size_t partitions = 1;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
- testServer.WaitInit("topic1");
-
-
- auto consumer = testServer.PQLib->CreateConsumer(MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false);
- UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError());
- auto isDead = consumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto msg1 = consumer->GetNextMessage();
- auto msg2 = consumer->GetNextMessage();
-
- testServer.PQLib = nullptr;
- Cerr << "PQLib destroyed" << Endl;
-
- UNIT_ASSERT(msg1.HasValue());
- UNIT_ASSERT(msg2.HasValue());
-
- UNIT_ASSERT(msg1.GetValue().Response.HasError());
- UNIT_ASSERT(msg2.GetValue().Response.HasError());
-
- auto msg3 = consumer->GetNextMessage();
- UNIT_ASSERT(msg3.HasValue());
- UNIT_ASSERT(msg3.GetValue().Response.HasError());
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp
deleted file mode 100644
index a9c9968081..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp
+++ /dev/null
@@ -1,349 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include "internals.h"
-
-#include <library/cpp/json/json_reader.h>
-#include <library/cpp/http/simple/http_client.h>
-#include <library/cpp/logger/priority.h>
-#include <library/cpp/tvmauth/client/facade.h>
-
-#include <util/generic/ptr.h>
-#include <util/string/builder.h>
-#include <util/system/env.h>
-#include <util/system/mutex.h>
-#include <library/cpp/string_utils/quote/quote.h>
-#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
-#include <kikimr/public/sdk/cpp/client/iam/iam.h>
-
-namespace NPersQueue {
-
-TString GetToken(ICredentialsProvider* credentials) {
- NPersQueue::TCredentials auth;
- credentials->FillAuthInfo(&auth);
- TString token;
- switch (auth.GetCredentialsCase()) {
- case NPersQueue::TCredentials::kTvmServiceTicket:
- token = auth.GetTvmServiceTicket();
- break;
- case NPersQueue::TCredentials::kOauthToken:
- token = auth.GetOauthToken();
- break;
- case NPersQueue::TCredentials::CREDENTIALS_NOT_SET:
- break;
- default:
- Y_VERIFY(true, "Unknown Credentials case.");
- }
- return token;
-}
-
-const TString& GetTvmPqServiceName() {
- static const TString PQ_SERVICE_NAME = "pq";
- return PQ_SERVICE_NAME;
-}
-
-class TInsecureCredentialsProvider : public ICredentialsProvider {
- void FillAuthInfo(NPersQueue::TCredentials*) const override {}
-};
-
-class TLogBridge: public NTvmAuth::ILogger {
-public:
- TLogBridge(TIntrusivePtr<NPersQueue::ILogger> logger = nullptr)
- : Logger(logger)
- {}
-
- void Log(int lvl, const TString& msg) override {
- if (Logger) {
- Logger->Log(msg, "", "", lvl);
- }
- }
-
-private:
- TIntrusivePtr<NPersQueue::ILogger> Logger;
-};
-
-class TTVMCredentialsProvider : public ICredentialsProvider {
-public:
- TTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr)
- : Alias(alias)
- {
- if (!settings.FetchServiceTicketsForDstsWithAliases.contains(Alias)) {
- ythrow yexception() << "alias for `" << Alias << "` must be set";
- }
- Logger = MakeIntrusive<TLogBridge>(std::move(logger));
- TvmClient = std::make_shared<NTvmAuth::TTvmClient>(settings, Logger);
- }
-
- TTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr)
- : TTVMCredentialsProvider(CreateSettings(secret, srcClientId, dstClientId, alias), alias, logger)
- {}
-
- TTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr)
- : TvmClient(tvmClient)
- , Alias(alias)
- , Logger(MakeIntrusive<TLogBridge>(std::move(logger)))
- {}
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override {
- try {
- auto status = TvmClient->GetStatus();
- if (status == NTvmAuth::TClientStatus::Ok) {
- authInfo->SetTvmServiceTicket(TvmClient->GetServiceTicketFor(Alias));
- } else {
- Logger->Error(TStringBuilder() << "Can't get ticket: " << status.GetLastError() << "\n");
- }
- } catch (...) {
- Logger->Error(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n");
- }
- }
-
-private:
- std::shared_ptr<NTvmAuth::TTvmClient> TvmClient;
- TString Alias;
- NTvmAuth::TLoggerPtr Logger;
-
- static NTvmAuth::NTvmApi::TClientSettings CreateSettings(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias) {
- NTvmAuth::NTvmApi::TClientSettings settings;
- settings.SetSelfTvmId(srcClientId);
- settings.EnableServiceTicketsFetchOptions(secret, {{alias, dstClientId}});
- return settings;
- }
-};
-
-class TTVMQloudCredentialsProvider : public ICredentialsProvider {
-public:
- TTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port)
- : HttpClient(TSimpleHttpClient("localhost", port))
- , Request(TStringBuilder() << "/tvm/tickets?src=" << CGIEscapeRet(srcAlias) << "&dsts=" << CGIEscapeRet(dstAlias))
- , DstAlias(dstAlias)
- , LastTicketUpdate(TInstant::Zero())
- , RefreshPeriod(refreshPeriod)
- , Logger(std::move(logger))
- , DstId(0)
- {
- GetTicket();
- }
-
- TTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port)
- : HttpClient(TSimpleHttpClient("localhost", port))
- , Request(TStringBuilder() << "/tvm/tickets?src=" << srcId << "&dsts=" << dstId)
- , LastTicketUpdate(TInstant::Zero())
- , RefreshPeriod(refreshPeriod)
- , Logger(std::move(logger))
- , DstId(dstId)
- {
- GetTicket();
- }
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override {
- if (TInstant::Now() > LastTicketUpdate + RefreshPeriod) {
- GetTicket();
- }
-
- authInfo->SetTvmServiceTicket(Ticket);
- }
-
-private:
- TSimpleHttpClient HttpClient;
- TString Request;
- TString DstAlias;
- mutable TString Ticket;
- mutable TInstant LastTicketUpdate;
- TDuration RefreshPeriod;
- TIntrusivePtr<ILogger> Logger;
- ui32 DstId;
-
- void GetTicket() const {
- try {
- TStringStream out;
- TSimpleHttpClient::THeaders headers;
- headers["Authorization"] = GetEnv("QLOUD_TVM_TOKEN");
- HttpClient.DoGet(Request, &out, headers);
- NJson::TJsonValue resp;
- NJson::ReadJsonTree(&out, &resp, true);
- TString localDstAlias = DstAlias;
- Y_VERIFY(!DstAlias.empty() || DstId != 0);
- Y_VERIFY(resp.GetMap().size() == 1);
- if (!localDstAlias.empty()) {
- if (!resp.Has(localDstAlias)) {
- ythrow yexception() << "Result doesn't contain dstAlias `" << localDstAlias << "`";
- }
- }
- else {
- for (const auto &[k, v] : resp.GetMap()) {
- if (!v.Has("tvm_id") || v["tvm_id"].GetIntegerSafe() != DstId)
- ythrow yexception() << "Result doesn't contain dstId `" << DstId << "`";
- localDstAlias = k;
- }
- }
- TString ticket = resp[localDstAlias]["ticket"].GetStringSafe();
- if (ticket.empty()) {
- ythrow yexception() << "Got empty ticket";
- }
- Ticket = ticket;
- LastTicketUpdate = TInstant::Now();
- } catch (...) {
- if (Logger) {
- Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR);
- }
- }
- }
-};
-
-class TOAuthCredentialsProvider : public ICredentialsProvider {
-public:
- TOAuthCredentialsProvider(const TString& token)
- : Token(token)
- {}
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override {
- authInfo->SetOauthToken(Token);
- }
-
-private:
- TString Token;
-};
-
-class TTVMCredentialsForwarder : public ITVMCredentialsForwarder {
-public:
- TTVMCredentialsForwarder() = default;
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override {
- TGuard<TSpinLock> guard(Lock);
- if (!Ticket.empty())
- authInfo->SetTvmServiceTicket(Ticket);
- }
-
- void SetTVMServiceTicket(const TString& ticket) override {
- TGuard<TSpinLock> guard(Lock);
- Ticket = ticket;
- }
-
-
-private:
- TString Ticket;
- TSpinLock Lock;
-};
-
-class IIAMCredentialsProviderWrapper : public ICredentialsProvider {
-public:
- IIAMCredentialsProviderWrapper(std::shared_ptr<NYdb::ICredentialsProviderFactory> factory, TIntrusivePtr<ILogger> logger = nullptr)
- : Provider(factory->CreateProvider())
- , Logger(std::move(logger))
- , Lock()
- {
- }
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override {
- TString ticket;
- try {
- with_lock(Lock) {
- ticket = Provider->GetAuthInfo();
- }
- } catch (...) {
- if (Logger) {
- Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR);
- }
- return;
- }
-
- authInfo->SetTvmServiceTicket(ticket);
- }
-
-private:
- std::shared_ptr<NYdb::ICredentialsProvider> Provider;
- TIntrusivePtr<ILogger> Logger;
- TMutex Lock;
-};
-
-class TIAMCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper {
-public:
- TIAMCredentialsProviderWrapper(TIntrusivePtr<ILogger> logger = nullptr)
- : IIAMCredentialsProviderWrapper(
- NYdb::CreateIamCredentialsProviderFactory(),
- std::move(logger)
- )
- {
- }
-};
-
-class TIAMJwtFileCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper {
-public:
- TIAMJwtFileCredentialsProviderWrapper(
- const TString& jwtKeyFilename, const TString& endpoint, const TDuration& refreshPeriod,
- const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr
- )
- : IIAMCredentialsProviderWrapper(
- NYdb::CreateIamJwtFileCredentialsProviderFactory(
- {{endpoint, refreshPeriod, requestTimeout}, jwtKeyFilename}),
- std::move(logger)
- )
- {}
-};
-
-class TIAMJwtParamsCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper {
-public:
- TIAMJwtParamsCredentialsProviderWrapper(
- const TString& jwtParams, const TString& endpoint, const TDuration& refreshPeriod,
- const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr
- )
- : IIAMCredentialsProviderWrapper(
- NYdb::CreateIamJwtParamsCredentialsProviderFactory(
- {{endpoint, refreshPeriod, requestTimeout}, jwtParams}),
- std::move(logger)
- )
- {}
-};
-
-std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider() {
- return std::make_shared<TInsecureCredentialsProvider>();
-}
-
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger) {
- return std::make_shared<TTVMCredentialsProvider>(secret, srcClientId, dstClientId, GetTvmPqServiceName(), std::move(logger));
-}
-
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger, const TString& alias) {
- return std::make_shared<TTVMCredentialsProvider>(settings, alias, std::move(logger));
-}
-
-std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger, const TString& alias) {
- return std::make_shared<TTVMCredentialsProvider>(tvmClient, alias, std::move(logger));
-}
-
-std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) {
- return std::make_shared<TTVMQloudCredentialsProvider>(srcAlias, dstAlias, refreshPeriod, std::move(logger), port);
-}
-
-std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) {
- return std::make_shared<TTVMQloudCredentialsProvider>(srcId, dstId, refreshPeriod, std::move(logger), port);
-}
-
-std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token) {
- return std::make_shared<TOAuthCredentialsProvider>(token);
-}
-
-std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder() {
- return std::make_shared<TTVMCredentialsForwarder>();
-}
-
-std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger) {
- return std::make_shared<TIAMCredentialsProviderWrapper>(logger);
-}
-
-std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder(
- const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger,
- const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout
-) {
- return std::make_shared<TIAMJwtFileCredentialsProviderWrapper>(jwtKeyFilename, endpoint,
- refreshPeriod, requestTimeout, logger);
-}
-
-std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder(
- const TString& jwtParams, TIntrusivePtr<ILogger> logger,
- const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout
-) {
- return std::make_shared<TIAMJwtParamsCredentialsProviderWrapper>(jwtParams, endpoint,
- refreshPeriod, requestTimeout, logger);
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp
deleted file mode 100644
index 0fbf62cf02..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp
+++ /dev/null
@@ -1,290 +0,0 @@
-#include "decompressing_consumer.h"
-#include "persqueue_p.h"
-
-#include <library/cpp/streams/lzop/lzop.h>
-#include <library/cpp/streams/zstd/zstd.h>
-
-#include <util/stream/mem.h>
-#include <util/stream/zlib.h>
-
-#include <atomic>
-
-namespace NPersQueue {
-
-TDecompressingConsumer::TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger)
- : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Logger(std::move(logger))
- , Subconsumer(std::move(subconsumer))
- , Settings(settings)
-{
-}
-
-void TDecompressingConsumer::Init() {
- DestroyedFuture = WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subconsumer->Destroyed());
-}
-
-TDecompressingConsumer::~TDecompressingConsumer() {
- DestroyQueue("Destructor called");
-}
-
-template <class T>
-void TDecompressingConsumer::SubscribeForQueueProcessing(NThreading::TFuture<T>& future) {
- const void* queueTag = this;
- std::weak_ptr<TDecompressingConsumer> self = shared_from_this();
- auto signalProcessQueue = [queueTag, self, pqLib = PQLib.Get()](const auto&) mutable {
- SignalProcessQueue(queueTag, std::move(self), pqLib);
- };
- future.Subscribe(signalProcessQueue);
-}
-
-NThreading::TFuture<TConsumerCreateResponse> TDecompressingConsumer::Start(TInstant deadline) noexcept {
- auto ret = Subconsumer->Start(deadline);
- SubscribeForQueueProcessing(ret); // when subconsumer starts, we will read ahead some messages
-
- // subscribe to death
- std::weak_ptr<TDecompressingConsumer> self = shared_from_this();
- auto isDeadHandler = [self](const auto& error) {
- auto selfShared = self.lock();
- if (selfShared && !selfShared->IsDestroyed) {
- selfShared->DestroyQueue(error.GetValue());
- }
- };
- PQLib->Subscribe(Subconsumer->IsDead(), this, isDeadHandler);
-
- return ret;
-}
-
-NThreading::TFuture<TError> TDecompressingConsumer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TDecompressingConsumer::Commit(const TVector<ui64>& cookies) noexcept {
- if (IsDestroyed) {
- ERR_LOG("Attempt to commit to dead consumer.", "", "");
- return;
- }
- return Subconsumer->Commit(cookies);
-}
-
-void TDecompressingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- if (IsDestroyed) {
- ERR_LOG("Attempt to request status from dead consumer.", "", "");
- return;
- }
- return Subconsumer->RequestPartitionStatus(topic, partition, generation);
-}
-
-
-NThreading::TFuture<void> TDecompressingConsumer::Destroyed() noexcept {
- return DestroyedFuture;
-}
-
-void TDecompressingConsumer::AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise) {
- Queue.emplace_back(promise);
- Queue.back().Future = Subconsumer->GetNextMessage();
- SubscribeForQueueProcessing(Queue.back().Future);
-}
-
-void TDecompressingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept {
- if (IsDestroyed) {
- TReadResponse response;
- auto* err = response.MutableError();
- err->SetCode(NErrorCode::ERROR);
- err->SetDescription("Consumer is dead.");
- promise.SetValue(TConsumerMessage(std::move(response)));
- return;
- }
- AddNewGetNextMessageRequest(promise);
-}
-
-void TDecompressingConsumer::ProcessQueue() {
- try {
- // stage 1: wait subconsumer answer and run decompression tasks
- for (TReadRequestInfo& request : Queue) {
- if (!request.Data && request.Future.HasValue()) {
- request.Data.ConstructInPlace(request.Future.ExtractValue());
- if (request.Data->Type == EMT_DATA) {
- RequestDecompressing(request);
- }
- }
- }
-
- // stage 2: answer ready tasks to client
- while (!Queue.empty()) {
- TReadRequestInfo& front = Queue.front();
- if (!front.Data || front.Data->Type == EMT_DATA && !front.AllDecompressing.HasValue()) {
- break;
- }
- if (front.Data->Type == EMT_DATA) {
- CopyDataToAnswer(front);
- }
-
- front.Promise.SetValue(TConsumerMessage(std::move(*front.Data)));
- Queue.pop_front();
- }
- } catch (const std::exception&) {
- DestroyQueue(TStringBuilder() << "Failed to decompress data: " << CurrentExceptionMessage());
- }
-}
-
-void TDecompressingConsumer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib) {
- auto processQueue = [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->ProcessQueue();
- }
- };
- Y_VERIFY(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue));
-}
-
-namespace {
-
-struct TWaitAll: public TAtomicRefCount<TWaitAll> {
- TWaitAll(size_t count)
- : Count(count)
- {
- }
-
- void OnResultReady() {
- if (--Count == 0) {
- Promise.SetValue();
- }
- }
-
- std::atomic<size_t> Count;
- NThreading::TPromise<void> Promise = NThreading::NewPromise<void>();
-};
-
-THolder<IInputStream> CreateDecompressor(const ECodec codec, IInputStream* origin) {
- THolder<IInputStream> result;
- if (codec == ECodec::GZIP) {
- result.Reset(new TZLibDecompress(origin));
- } else if (codec == ECodec::LZOP) {
- result.Reset(new TLzopDecompress(origin));
- } else if (codec == ECodec::ZSTD) {
- result.Reset(new TZstdDecompress(origin));
- }
- return result;
-}
-
-} // namespace
-
-void TDecompressingConsumer::RequestDecompressing(TReadRequestInfo& request) {
- size_t futuresCount = 0;
- const TReadResponse::TData& data = request.Data->Response.GetData();
- request.BatchFutures.resize(data.MessageBatchSize());
- for (size_t i = 0; i < data.MessageBatchSize(); ++i) {
- const TReadResponse::TData::TMessageBatch& batch = data.GetMessageBatch(i);
- request.BatchFutures.reserve(batch.MessageSize());
- for (const TReadResponse::TData::TMessage& message : batch.GetMessage()) {
- if (message.GetMeta().GetCodec() != ECodec::RAW) {
- ++futuresCount;
- request.BatchFutures[i].push_back(RequestDecompressing(message));
- }
- }
- }
-
- if (futuresCount == 0) {
- request.AllDecompressing = NThreading::MakeFuture(); // done
- return;
- } else {
- TIntrusivePtr<TWaitAll> waiter = new TWaitAll(futuresCount);
- auto handler = [waiter](const auto&) {
- waiter->OnResultReady();
- };
- for (auto& batch : request.BatchFutures) {
- for (auto& future : batch) {
- future.Subscribe(handler);
- }
- }
- request.AllDecompressing = waiter->Promise.GetFuture();
- }
- SubscribeForQueueProcessing(request.AllDecompressing);
-}
-
-NThreading::TFuture<TString> TDecompressingConsumer::RequestDecompressing(const TReadResponse::TData::TMessage& message) {
- TString data = message.GetData();
- ECodec codec = message.GetMeta().GetCodec();
- NThreading::TPromise<TString> promise = NThreading::NewPromise<TString>();
- auto decompress = [data, codec, promise]() mutable {
- Decompress(data, codec, promise);
- };
- Y_VERIFY(PQLib->GetCompressionPool().AddFunc(decompress));
- return promise.GetFuture();
-}
-
-void TDecompressingConsumer::Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise) {
- try {
- // TODO: Check if decompression was successfull, i.e. if 'data' is valid byte array compressed with 'codec'
- TMemoryInput iss(data.data(), data.size());
- THolder<IInputStream> dec = CreateDecompressor(codec, &iss);
- if (!dec) {
- ythrow yexception() << "Failed to create decompressor";
- }
-
- TString result;
- TStringOutput oss(result);
- TransferData(dec.Get(), &oss);
-
- promise.SetValue(result);
- } catch (...) {
- promise.SetException(std::current_exception());
- }
-}
-
-void TDecompressingConsumer::CopyDataToAnswer(TReadRequestInfo& request) {
- TReadResponse::TData& data = *request.Data->Response.MutableData();
- Y_VERIFY(request.BatchFutures.size() == data.MessageBatchSize());
- for (size_t i = 0; i < request.BatchFutures.size(); ++i) {
- TReadResponse::TData::TMessageBatch& batch = *data.MutableMessageBatch(i);
- auto currentMessage = request.BatchFutures[i].begin();
- for (TReadResponse::TData::TMessage& message : *batch.MutableMessage()) {
- if (message.GetMeta().GetCodec() != ECodec::RAW) {
- Y_VERIFY(currentMessage != request.BatchFutures[i].end());
- try {
- message.SetData(currentMessage->GetValue());
- message.MutableMeta()->SetCodec(ECodec::RAW);
- } catch (const std::exception& ex) {
- if (Settings.SkipBrokenChunks) {
- message.SetBrokenPackedData(message.GetData());
- message.SetData("");
- } else {
- throw;
- }
- }
- ++currentMessage;
- }
- }
- Y_VERIFY(currentMessage == request.BatchFutures[i].end());
- }
-}
-
-void TDecompressingConsumer::DestroyQueue(const TString& errorMessage) {
- TError error;
- error.SetDescription(errorMessage);
- error.SetCode(NErrorCode::ERROR);
- DestroyQueue(error);
-}
-
-void TDecompressingConsumer::DestroyQueue(const TError& error) {
- if (IsDestroyed) {
- return;
- }
-
- for (TReadRequestInfo& request : Queue) {
- TReadResponse response;
- *response.MutableError() = error;
- request.Promise.SetValue(TConsumerMessage(std::move(response)));
- }
- Queue.clear();
- IsDeadPromise.SetValue(error);
- IsDestroyed = true;
-
- DestroyPQLibRef();
-}
-
-void TDecompressingConsumer::Cancel() {
- DestroyQueue(GetCancelReason());
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h
deleted file mode 100644
index c547ef8b3a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h
+++ /dev/null
@@ -1,77 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include "internals.h"
-#include "iconsumer_p.h"
-
-#include <util/generic/maybe.h>
-
-#include <deque>
-#include <memory>
-#include <vector>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class TDecompressingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TDecompressingConsumer> {
-public:
- using IConsumerImpl::GetNextMessage;
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override;
- NThreading::TFuture<TError> IsDead() noexcept override;
- NThreading::TFuture<void> Destroyed() noexcept override;
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override;
- void Commit(const TVector<ui64>& cookies) noexcept override;
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override;
- void Init() override;
-
- ~TDecompressingConsumer();
- TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger);
-
- void Cancel() override;
-
-private:
- struct TReadRequestInfo {
- TReadRequestInfo() = default;
-
- TReadRequestInfo(const NThreading::TPromise<TConsumerMessage>& promise)
- : Promise(promise)
- {
- }
-
- NThreading::TPromise<TConsumerMessage> Promise;
- NThreading::TFuture<TConsumerMessage> Future;
- TMaybe<TConsumerMessage> Data;
-
- using TMessageFutures = std::vector<NThreading::TFuture<TString>>;
- using TBatchFutures = std::vector<TMessageFutures>;
- TBatchFutures BatchFutures;
- NThreading::TFuture<void> AllDecompressing;
- };
-
- void ProcessQueue();
- static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib);
- void DestroyQueue(const TString& errorMessage);
- void DestroyQueue(const TError& error);
-
- template <class T>
- void SubscribeForQueueProcessing(NThreading::TFuture<T>& future);
-
- void RequestDecompressing(TReadRequestInfo& request);
- void CopyDataToAnswer(TReadRequestInfo& request);
- NThreading::TFuture<TString> RequestDecompressing(const TReadResponse::TData::TMessage& message);
- static void Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise);
- void AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise);
-
-protected:
- TIntrusivePtr<ILogger> Logger;
- std::shared_ptr<IConsumerImpl> Subconsumer;
- std::deque<TReadRequestInfo> Queue;
- TConsumerSettings Settings;
- NThreading::TFuture<void> DestroyedFuture; // Subconsumer may be deleted, so store our future here.
- NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>();
- bool IsDestroyed = false;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp
deleted file mode 100644
index 6c22163e8f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp
+++ /dev/null
@@ -1,265 +0,0 @@
-#include "decompressing_consumer.h"
-#include "local_caller.h"
-#include "persqueue_p.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/gmock_in_unittest/gmock.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace testing;
-
-namespace NPersQueue {
-
-class TMockConsumer: public IConsumerImpl {
-public:
- TMockConsumer()
- : IConsumerImpl(nullptr, nullptr)
- {
- }
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override {
- return NThreading::MakeFuture<TConsumerCreateResponse>(MockStart());
- }
-
- NThreading::TFuture<TError> IsDead() noexcept override {
- return MockIsDead();
- }
-
- NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override {
- return NThreading::MakeFuture<TConsumerMessage>(MockGetNextMessage());
- }
-
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override {
- promise.SetValue(MockGetNextMessage());
- }
-
- void Commit(const TVector<ui64>&) noexcept override {
- }
-
- void RequestPartitionStatus(const TString&, ui64, ui64) noexcept override {
- }
-
- void Cancel() override {
- }
-
- MOCK_METHOD(TConsumerCreateResponse, MockStart, (), ());
- MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ());
- MOCK_METHOD(TConsumerMessage, MockGetNextMessage, (), ());
-};
-
-template <class TMock = TMockConsumer>
-struct TDecompressingConsumerBootstrap {
- ~TDecompressingConsumerBootstrap() {
- Lib->CancelObjectsAndWait();
- }
-
- void Create() {
- Lib = new TPQLibPrivate(PQLibSettings);
- MockConsumer = std::make_shared<TMock>();
- DecompressingConsumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(MockConsumer, Settings, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger);
- Lib->AddToDestroySet(DecompressingConsumer);
- }
-
- void MakeOKStartResponse() {
- TReadResponse rresp;
- rresp.MutableInit()->SetSessionId("test-session");
- TConsumerCreateResponse resp(std::move(rresp));
- EXPECT_CALL(*MockConsumer, MockStart())
- .WillOnce(Return(resp));
- }
-
- void ExpectIsDeadCall() {
- EXPECT_CALL(*MockConsumer, MockIsDead())
- .WillOnce(Return(DeadPromise.GetFuture()));
- }
-
- void Start() {
- MakeOKStartResponse();
- UNIT_ASSERT_STRINGS_EQUAL(DecompressingConsumer->Start().GetValueSync().Response.GetInit().GetSessionId(), "test-session");
- }
-
- TConsumerSettings Settings;
- TPQLibSettings PQLibSettings;
- TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG);
- TIntrusivePtr<TPQLibPrivate> Lib;
- std::shared_ptr<TMock> MockConsumer;
- std::shared_ptr<IConsumerImpl> DecompressingConsumer;
- NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>();
-};
-
-Y_UNIT_TEST_SUITE(TDecompressingConsumerTest) {
- Y_UNIT_TEST(DiesOnDeadSubconsumer) {
- TDecompressingConsumerBootstrap<> bootstrap;
- bootstrap.Create();
- bootstrap.ExpectIsDeadCall();
- bootstrap.Start();
-
- auto isDead = bootstrap.DecompressingConsumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- bootstrap.DeadPromise.SetValue(TError());
-
- isDead.GetValueSync(); // doesn't hang on
- }
-
- Y_UNIT_TEST(PassesNonData) {
- TDecompressingConsumerBootstrap<> bootstrap;
- bootstrap.Create();
- bootstrap.ExpectIsDeadCall();
- bootstrap.Start();
-
- InSequence sequence;
- {
- TReadResponse resp;
- resp.MutableCommit()->AddCookie(42);
-
- TConsumerMessage ret(std::move(resp));
-
- EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage())
- .WillOnce(Return(ret));
- }
-
- {
- TReadResponse resp;
- resp.MutableRelease()->SetTopic("topic!");
-
- TConsumerMessage ret(std::move(resp));
-
- EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage())
- .WillOnce(Return(ret));
- }
-
- auto passed1 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().CookieSize(), 1, passed1.Response);
- UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().GetCookie(0), 42, passed1.Response);
-
- auto passed2 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync();
- UNIT_ASSERT_STRINGS_EQUAL_C(passed2.Response.GetRelease().GetTopic(), "topic!", passed2.Response);
- }
-
- void ProcessesBrokenChunks(bool skip) {
- TDecompressingConsumerBootstrap<> bootstrap;
- bootstrap.Settings.SkipBrokenChunks = skip;
- bootstrap.Create();
- bootstrap.ExpectIsDeadCall();
- bootstrap.Start();
-
- TReadResponse resp;
- auto* msg = resp.MutableData()->AddMessageBatch()->AddMessage();
- msg->SetData("hjdhkjhkjhshqsiuhqisuqihsi;");
- msg->MutableMeta()->SetCodec(ECodec::LZOP);
-
- EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage())
- .WillOnce(Return(TConsumerMessage(std::move(resp))));
-
- auto isDead = bootstrap.DecompressingConsumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto passed = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync();
-
- if (skip) {
- UNIT_ASSERT(!passed.Response.HasError());
- isDead.HasValue();
- UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().MessageBatchSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().GetMessageBatch(0).MessageSize(), 1);
- UNIT_ASSERT(passed.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData().empty());
- } else {
- UNIT_ASSERT(passed.Response.HasError());
- isDead.GetValueSync();
-
- UNIT_ASSERT(bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync().Response.HasError());
- }
-
- //DestroyAndWait(bootstrap.DecompressingConsumer);
- }
-
- Y_UNIT_TEST(DiesOnBrokenChunks) {
- ProcessesBrokenChunks(false);
- }
-
- Y_UNIT_TEST(SkipsBrokenChunks) {
- ProcessesBrokenChunks(true);
- }
-
- static void AddMessage(TReadResponse::TData::TMessageBatch* batch, const TString& sourceData, ECodec codec = ECodec::RAW) {
- auto* msg = batch->AddMessage();
- msg->MutableMeta()->SetCodec(codec);
- if (codec == ECodec::RAW) {
- msg->SetData(sourceData);
- } else {
- msg->SetData(TData::Encode(sourceData, codec, -1).GetEncodedData());
- }
- }
-
- Y_UNIT_TEST(DecompessesDataInProperChuncksOrder) {
- TDecompressingConsumerBootstrap<> bootstrap;
- bootstrap.Create();
- bootstrap.ExpectIsDeadCall();
- bootstrap.Start();
-
- InSequence sequence;
- {
- TReadResponse resp;
- auto* batch = resp.MutableData()->AddMessageBatch();
- AddMessage(batch, "message1", ECodec::LZOP);
- AddMessage(batch, "message2");
- AddMessage(batch, "message3", ECodec::GZIP);
-
- resp.MutableData()->AddMessageBatch();
-
- batch = resp.MutableData()->AddMessageBatch();
- AddMessage(batch, "messageA", ECodec::LZOP);
- AddMessage(batch, "messageB", ECodec::ZSTD);
- AddMessage(batch, "messageC");
-
- EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage())
- .WillOnce(Return(TConsumerMessage(std::move(resp))));
- }
-
- {
- TReadResponse resp;
- AddMessage(resp.MutableData()->AddMessageBatch(), "trololo", ECodec::LZOP);
-
- EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage())
- .WillOnce(Return(TConsumerMessage(std::move(resp))));
- }
-
- auto isDead = bootstrap.DecompressingConsumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto f1 = bootstrap.DecompressingConsumer->GetNextMessage();
- auto f2 = bootstrap.DecompressingConsumer->GetNextMessage();
-
- auto data1 = f1.GetValueSync().Response.GetData();
- auto data2 = f2.GetValueSync().Response.GetData();
-
- UNIT_ASSERT_VALUES_EQUAL(data1.MessageBatchSize(), 3);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).MessageSize(), 3);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(1).MessageSize(), 0);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).MessageSize(), 3);
-
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetData(), "message1");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetData(), "message2");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetData(), "message3");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW);
-
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetData(), "messageA");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetData(), "messageB");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW);
- UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetData(), "messageC");
- UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW);
-
-
- UNIT_ASSERT_VALUES_EQUAL(data2.MessageBatchSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).MessageSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetData(), "trololo");
- UNIT_ASSERT_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW);
-
-
- UNIT_ASSERT(!isDead.HasValue());
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp
deleted file mode 100644
index 2ebfcd8d07..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#include "iconsumer_p.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-TPublicConsumer::TPublicConsumer(std::shared_ptr<IConsumerImpl> impl)
- : Impl(std::move(impl))
-{
-}
-
-TPublicConsumer::~TPublicConsumer() {
- Impl->Cancel();
-}
-
-IConsumerImpl::IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib)
- : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib))
-{
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h
deleted file mode 100644
index 35c151b9de..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h
+++ /dev/null
@@ -1,51 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h>
-#include "interface_common.h"
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class IConsumerImpl: public IConsumer, public TSyncDestroyed {
-public:
- IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib);
-
- virtual void Init() {
- }
-
- using IConsumer::GetNextMessage;
- virtual void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept = 0;
-};
-
-// Consumer that is given to client.
-class TPublicConsumer: public IConsumer {
-public:
- explicit TPublicConsumer(std::shared_ptr<IConsumerImpl> impl);
- ~TPublicConsumer();
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override {
- return Impl->Start(deadline);
- }
-
- NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override {
- return Impl->GetNextMessage();
- }
-
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override {
- Impl->RequestPartitionStatus(topic, partition, generation);
- }
-
- void Commit(const TVector<ui64>& cookies) noexcept override {
- Impl->Commit(cookies);
- }
-
- NThreading::TFuture<TError> IsDead() noexcept override {
- return Impl->IsDead();
- }
-
-private:
- std::shared_ptr<IConsumerImpl> Impl;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp
deleted file mode 100644
index e88c5b1280..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-#include "interface_common.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-const TString& GetCancelReason()
-{
- static const TString reason = "Destroyed";
- return reason;
-}
-
-TSyncDestroyed::TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib)
- : DestroyEventRef(std::move(destroyEventRef))
- , PQLib(std::move(pqLib))
-{
-}
-
-TSyncDestroyed::~TSyncDestroyed() {
- DestroyedPromise.SetValue();
-}
-
-void TSyncDestroyed::DestroyPQLibRef() {
- auto guard = Guard(DestroyLock);
- if (DestroyEventRef) {
- IsCanceling = true;
- PQLib = nullptr;
- DestroyEventRef = nullptr;
- }
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h
deleted file mode 100644
index bd3a228b2c..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/ptr.h>
-#include <util/system/spinlock.h>
-
-#include <memory>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-const TString& GetCancelReason();
-
-class TSyncDestroyed {
-protected:
- TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib);
-
-public:
- // Get future that is signalled after object's destructor.
- // Non-pqlib threads can wait on this future to ensure that
- // all async operations are finished.
- virtual NThreading::TFuture<void> Destroyed() noexcept {
- return DestroyedPromise.GetFuture();
- }
-
- // This destructor will be executed after real object's destructor
- // in which it is expected to set all owned promises.
- virtual ~TSyncDestroyed();
-
- virtual void Cancel() = 0;
-
- void SetDestroyEventRef(std::shared_ptr<void> ref) {
- DestroyEventRef = std::move(ref);
- }
-
-protected:
- // Needed for proper PQLib deinitialization
- void DestroyPQLibRef();
-
-protected:
- NThreading::TPromise<void> DestroyedPromise = NThreading::NewPromise<void>();
- std::shared_ptr<void> DestroyEventRef;
- TIntrusivePtr<TPQLibPrivate> PQLib;
- bool IsCanceling = false;
- TAdaptiveLock DestroyLock;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h
deleted file mode 100644
index 5c893da75e..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h
+++ /dev/null
@@ -1,105 +0,0 @@
-#pragma once
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-#include <library/cpp/grpc/common/time_point.h>
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-#include <util/string/cast.h>
-
-#include <grpc++/channel.h>
-#include <grpc++/create_channel.h>
-
-#include <contrib/libs/grpc/include/grpcpp/impl/codegen/client_context.h>
-
-#include <chrono>
-
-#define WRITE_LOG(msg, srcId, sessionId, level, logger) \
- if (logger && logger->IsEnabled(level)) { \
- logger->Log(TStringBuilder() << msg, srcId, sessionId, level); \
- }
-
-#define DEBUG_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_DEBUG, Logger)
-#define INFO_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_INFO, Logger)
-#define WARN_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_WARNING, Logger)
-#define ERR_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_ERR, Logger)
-
-namespace NPersQueue {
-
-TString GetToken(ICredentialsProvider* credentials);
-
-void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials);
-
-bool UseCDS(const TServerSetting& server);
-
-struct TWriteData {
- TProducerSeqNo SeqNo;
- TData Data;
-
- TWriteData(ui64 seqNo, TData&& data)
- : SeqNo(seqNo)
- , Data(std::move(data))
- {}
-};
-
-class IQueueEvent {
-public:
- virtual ~IQueueEvent() = default;
-
- //! Execute an action defined by implementation.
- virtual bool Execute(bool ok) = 0;
-
- //! Finish and destroy request.
- virtual void DestroyRequest() = 0;
-};
-
-class IHandler : public TAtomicRefCount<IHandler> {
-public:
- IHandler()
- {}
-
- virtual ~IHandler()
- {}
-
- virtual void Destroy(const TError&) = 0;
- virtual void Done() = 0;
- virtual TString ToString() = 0;
-};
-
-using IHandlerPtr = TIntrusivePtr<IHandler>;
-
-class TQueueEvent : public IQueueEvent {
-public:
- TQueueEvent(IHandlerPtr handler)
- : Handler(std::move(handler))
- {
- Y_ASSERT(Handler);
- }
-
-private:
- bool Execute(bool ok) override {
- if (ok) {
- Handler->Done();
- } else {
- TError error;
- error.SetDescription("event " + Handler->ToString() + " failed");
- error.SetCode(NErrorCode::ERROR);
- Handler->Destroy(error);
- }
- return false;
- }
-
- virtual ~TQueueEvent() {
- Handler.Reset();
- }
-
- void DestroyRequest() override
- {
- delete this;
- }
-
- IHandlerPtr Handler;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp
deleted file mode 100644
index f245df74fe..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#include "iprocessor_p.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-TPublicProcessor::TPublicProcessor(std::shared_ptr<IProcessorImpl> impl)
- : Impl(std::move(impl))
-{
-}
-
-TPublicProcessor::~TPublicProcessor() {
- Impl->Cancel();
-}
-
-IProcessorImpl::IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib)
- : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib))
-{
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h
deleted file mode 100644
index e315b8412f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h>
-#include "interface_common.h"
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-struct IProcessorImpl: public IProcessor, public TSyncDestroyed {
- using IProcessor::GetNextData;
-
- IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib);
-
- // Initialization after constructor (for example, for correct call of shared_from_this())
- virtual void Init() {
- }
- virtual void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept = 0;
-};
-
-// Processor that is given to client.
-class TPublicProcessor: public IProcessor {
-public:
- explicit TPublicProcessor(std::shared_ptr<IProcessorImpl> impl);
- ~TPublicProcessor();
-
- NThreading::TFuture<TOriginData> GetNextData() noexcept override {
- return Impl->GetNextData();
- }
-
-private:
- std::shared_ptr<IProcessorImpl> Impl;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp
deleted file mode 100644
index c7a267b862..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#include "iproducer_p.h"
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-TPublicProducer::TPublicProducer(std::shared_ptr<IProducerImpl> impl)
- : Impl(std::move(impl))
-{
-}
-
-TPublicProducer::~TPublicProducer() {
- Impl->Cancel();
-}
-
-IProducerImpl::IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib)
- : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib))
-{
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h
deleted file mode 100644
index 437798c551..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include "interface_common.h"
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-struct IProducerImpl: public IProducer, public TSyncDestroyed {
- using IProducer::Write;
-
- IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib);
-
- // Initialization after constructor (for example, for correct call of shared_from_this())
- virtual void Init() {
- }
- virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept = 0;
- virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept = 0;
-
- NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData) noexcept override {
- Y_FAIL("");
- return NThreading::NewPromise<TProducerCommitResponse>().GetFuture();
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TData) noexcept override {
- Y_FAIL("");
- return NThreading::NewPromise<TProducerCommitResponse>().GetFuture();
- }
-};
-
-// Producer that is given to client.
-class TPublicProducer: public IProducer {
-public:
- explicit TPublicProducer(std::shared_ptr<IProducerImpl> impl);
- ~TPublicProducer();
-
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept {
- return Impl->Start(deadline);
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept {
- return Impl->Write(seqNo, std::move(data));
- }
-
- NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept {
- return Impl->Write(std::move(data));
- }
-
- NThreading::TFuture<TError> IsDead() noexcept {
- return Impl->IsDead();
- }
-
-private:
- std::shared_ptr<IProducerImpl> Impl;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h
deleted file mode 100644
index 2ac0b3ec18..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h
+++ /dev/null
@@ -1,303 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include "interface_common.h"
-#include "internals.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include "persqueue_p.h"
-
-namespace NPersQueue {
-
-template <class TImpl>
-class TCancelCaller: public TImpl {
-public:
- using TImpl::TImpl;
-
- std::shared_ptr<TCancelCaller> MakeSharedRef() {
- return std::static_pointer_cast<TCancelCaller>(this->shared_from_this());
- }
-
- void Cancel() override {
- auto guard = Guard(this->DestroyLock);
- if (!this->IsCanceling) {
- this->IsCanceling = true;
- AddCancelCaller();
- }
- }
-
-protected:
- const TImpl* QueueTag() const { // we must have exactly the same thread tag as TImpl object
- return this;
- }
-
- static void MakeCanceledError(TError& error) {
- error.SetDescription(GetCancelReason());
- error.SetCode(NErrorCode::ERROR);
- }
-
- template <class TResponseProto>
- static void MakeCanceledError(TResponseProto& resp) {
- MakeCanceledError(*resp.MutableError());
- }
-
- template <class TResponse, class... TArgs>
- static void MakeCanceledError(NThreading::TPromise<TResponse>& promise, TArgs&&...) { // the most common case for calls
- decltype(promise.GetValue().Response) responseProto;
- MakeCanceledError(responseProto);
- promise.SetValue(TResponse(std::move(responseProto)));
- }
-
- template <class TResponse>
- static TResponse MakeCanceledError() {
- TResponse resp;
- MakeCanceledError(*resp.MutableError());
- return resp;
- }
-
-private:
- void AddCancelCaller() {
- std::weak_ptr<TCancelCaller> selfWeak = MakeSharedRef();
- auto caller = [selfWeak]() mutable {
- auto selfShared = selfWeak.lock();
- if (selfShared) {
- selfShared->TImpl::Cancel();
- }
- };
- Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(QueueTag()).AddFunc(caller));
- }
-};
-
-template <class TImpl, class TStartResponse>
-class TLocalStartDeadImplCaller: public TCancelCaller<TImpl> {
-protected:
- using TCancelCaller<TImpl>::TCancelCaller;
-
- NThreading::TFuture<TStartResponse> Start(TInstant deadline) noexcept override {
- IsDead(); // subscribe for impl future
- auto guard = Guard(this->DestroyLock);
- if (this->IsCanceling) {
- guard.Release();
- decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto;
- this->MakeCanceledError(respProto);
- return NThreading::MakeFuture(TStartResponse(std::move(respProto)));
- }
- if (StartPromise.Initialized()) {
- guard.Release();
- decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto;
- TError& error = *respProto.MutableError();
- error.SetDescription("Start was already called");
- error.SetCode(NErrorCode::BAD_REQUEST);
- return NThreading::MakeFuture(TStartResponse(std::move(respProto)));
- }
- StartPromise = NThreading::NewPromise<TStartResponse>();
- AddStartCaller(deadline);
- return StartPromise.GetFuture();
- }
-
- NThreading::TFuture<TError> IsDead() noexcept override {
- auto guard = Guard(this->DestroyLock);
- if (this->IsCanceling && !IsDeadCalled) {
- guard.Release();
- TError err;
- this->MakeCanceledError(err);
- return NThreading::MakeFuture(err);
- }
- if (!IsDeadCalled) {
- AddIsDeadCaller();
- }
- return DeadPromise.GetFuture();
- }
-
-private:
- std::shared_ptr<TLocalStartDeadImplCaller> MakeSharedRef() {
- return std::static_pointer_cast<TLocalStartDeadImplCaller>(this->shared_from_this());
- }
-
- void AddIsDeadCaller() {
- IsDeadCalled = true;
- std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef();
- auto isDeadCaller = [selfShared]() mutable {
- selfShared->CallIsDead();
- };
- Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(isDeadCaller));
- }
-
- void CallIsDead() {
- auto deadPromise = DeadPromise;
- TImpl::IsDead().Subscribe([deadPromise](const auto& future) mutable {
- deadPromise.SetValue(future.GetValue());
- });
- }
-
- void AddStartCaller(TInstant deadline) {
- std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef();
- auto startCaller = [selfShared, deadline]() mutable {
- selfShared->CallStart(deadline);
- };
- Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(startCaller));
- }
-
- void CallStart(TInstant deadline) {
- auto startPromise = StartPromise;
- TImpl::Start(deadline).Subscribe([startPromise](const auto& future) mutable {
- startPromise.SetValue(future.GetValue());
- });
- }
-
-private:
- NThreading::TPromise<TStartResponse> StartPromise;
- NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>();
- bool IsDeadCalled = false;
-};
-
-#define LOCAL_CALLER(Func, Args, DeclTail, ...) \
- void Func Args DeclTail { \
- auto guard = Guard(this->DestroyLock); \
- if (this->IsCanceling) { \
- guard.Release(); \
- this->MakeCanceledError(__VA_ARGS__); \
- return; \
- } \
- auto selfShared = MakeSharedRef(); \
- auto caller = [selfShared, __VA_ARGS__]() mutable { \
- selfShared->TImpl::Func(__VA_ARGS__); \
- }; \
- Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(caller));\
- }
-
-// Assumes that there is function with promise interface
-// first arg must be promise
-#define LOCAL_CALLER_WITH_FUTURE(ReturnType, Func, Args, DeclTail, ...) \
- NThreading::TFuture<ReturnType> Func Args DeclTail { \
- auto promise = NThreading::NewPromise<ReturnType>(); \
- Func(__VA_ARGS__); \
- return promise.GetFuture(); \
- }
-
-// Helper classes for producer/consumer/processor impls.
-// Wrappers that delegate calls to its methods to proper thread pool local threads.
-// The result is that all calls to impl are serialized and locks are no more needed.
-//
-// Implies:
-// That impl class is std::enable_shared_from_this descendant.
-
-template <class TImpl>
-class TLocalProducerImplCaller: public TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse> {
-public:
- using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::TLocalStartDeadImplCaller;
-
- LOCAL_CALLER(Write,
- (NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data),
- noexcept override,
- promise, seqNo, data)
-
- LOCAL_CALLER(Write,
- (NThreading::TPromise<TProducerCommitResponse>& promise, TData data),
- noexcept override,
- promise, data)
-
- LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse,
- Write,
- (TProducerSeqNo seqNo, TData data),
- noexcept override,
- promise, seqNo, data)
-
- LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse,
- Write,
- (TData data),
- noexcept override,
- promise, data)
-
-protected:
- using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::MakeCanceledError;
- static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) {
- TWriteResponse responseProto;
- MakeCanceledError(responseProto);
- promise.SetValue(TProducerCommitResponse(seqNo, std::move(data), std::move(responseProto)));
- }
-
- static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) {
- MakeCanceledError(promise, TProducerSeqNo(0), std::move(data));
- }
-
-private:
- std::shared_ptr<TLocalProducerImplCaller> MakeSharedRef() {
- return std::static_pointer_cast<TLocalProducerImplCaller>(this->shared_from_this());
- }
-};
-
-template <class TImpl>
-class TLocalConsumerImplCaller: public TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse> {
-public:
- using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::TLocalStartDeadImplCaller;
-
- LOCAL_CALLER(GetNextMessage,
- (NThreading::TPromise<TConsumerMessage>& promise),
- noexcept override,
- promise)
-
- LOCAL_CALLER_WITH_FUTURE(TConsumerMessage,
- GetNextMessage,
- (),
- noexcept override,
- promise)
-
- LOCAL_CALLER(Commit,
- (const TVector<ui64>& cookies),
- noexcept override,
- cookies)
-
- LOCAL_CALLER(RequestPartitionStatus,
- (const TString& topic, ui64 partition, ui64 generation),
- noexcept override,
- topic, partition, generation)
-
-private:
- using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::MakeCanceledError;
- static void MakeCanceledError(const TVector<ui64>& cookies) {
- Y_UNUSED(cookies);
- // Do nothing, because this method doesn't return future
- }
-
- static void MakeCanceledError(const TString&, ui64, ui64) {
- // Do nothing, because this method doesn't return future
- }
-
-
- std::shared_ptr<TLocalConsumerImplCaller> MakeSharedRef() {
- return std::static_pointer_cast<TLocalConsumerImplCaller>(this->shared_from_this());
- }
-};
-
-template <class TImpl>
-class TLocalProcessorImplCaller: public TCancelCaller<TImpl> {
-public:
- using TCancelCaller<TImpl>::TCancelCaller;
-
- LOCAL_CALLER(GetNextData,
- (NThreading::TPromise<TOriginData>& promise),
- noexcept override,
- promise)
-
- LOCAL_CALLER_WITH_FUTURE(TOriginData,
- GetNextData,
- (),
- noexcept override,
- promise)
-
-private:
- using TCancelCaller<TImpl>::MakeCanceledError;
- static void MakeCanceledError(NThreading::TPromise<TOriginData>& promise) {
- promise.SetValue(TOriginData()); // with empty data
- }
-
- std::shared_ptr<TLocalProcessorImplCaller> MakeSharedRef() {
- return std::static_pointer_cast<TLocalProcessorImplCaller>(this->shared_from_this());
- }
-};
-
-#undef LOCAL_CALLER
-#undef LOCAL_CALLER_WITH_FUTURE
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp
deleted file mode 100644
index 98d44d14c5..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h>
-#include <util/datetime/base.h>
-#include <util/string/builder.h>
-#include <util/stream/str.h>
-
-namespace NPersQueue {
-
-static const TStringBuf LogLevelsStrings[] = {
- "EMERG",
- "ALERT",
- "CRITICAL_INFO",
- "ERROR",
- "WARNING",
- "NOTICE",
- "INFO",
- "DEBUG",
-};
-
-TStringBuf TCerrLogger::LevelToString(int level) {
- return LogLevelsStrings[ClampVal(level, 0, int(Y_ARRAY_SIZE(LogLevelsStrings) - 1))];
-}
-
-void TCerrLogger::Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) {
- if (level > Level) {
- return;
- }
-
- TStringBuilder message;
- message << TInstant::Now() << " :" << LevelToString(level) << ":";
- if (sourceId) {
- message << " SourceId [" << sourceId << "]:";
- }
- if (sessionId) {
- message << " SessionId [" << sessionId << "]:";
- }
- message << " " << msg << "\n";
- Cerr << message;
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp
deleted file mode 100644
index cd986532dc..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp
+++ /dev/null
@@ -1,645 +0,0 @@
-#include "multicluster_consumer.h"
-#include "persqueue_p.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <library/cpp/containers/intrusive_rb_tree/rb_tree.h>
-
-#include <util/generic/guid.h>
-#include <util/generic/strbuf.h>
-
-namespace NPersQueue {
-
-struct TMultiClusterConsumer::TSubconsumerInfo {
- TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer);
-
- std::shared_ptr<IConsumerImpl> Consumer;
- NThreading::TFuture<TConsumerCreateResponse> StartFuture;
- NThreading::TFuture<TError> DeadFuture;
- bool StartResponseWasGot = false; // We call methods of this consumer only after this flag was set.
-
- std::deque<NThreading::TFuture<TConsumerMessage>> PendingRequests;
-};
-
-namespace {
-
-struct TConsumerCookieKey;
-struct TUserCookieMappingItem;
-struct TConsumerCookieMappingItem;
-
-struct TConsumerCookieKey {
- size_t Subconsumer = 0;
- ui64 ConsumerCookie = 0;
-
- TConsumerCookieKey(size_t subconsumer, ui64 consumerCookie)
- : Subconsumer(subconsumer)
- , ConsumerCookie(consumerCookie)
- {
- }
-
- bool operator<(const TConsumerCookieKey& other) const {
- return std::make_pair(Subconsumer, ConsumerCookie) < std::make_pair(other.Subconsumer, other.ConsumerCookie);
- }
-};
-
-struct TUserCookieCmp {
- static ui64 GetKey(const TUserCookieMappingItem& item);
-
- static bool Compare(ui64 l, const TUserCookieMappingItem& r) {
- return l < GetKey(r);
- }
-
- static bool Compare(const TUserCookieMappingItem& l, ui64 r) {
- return GetKey(l) < r;
- }
-
- static bool Compare(const TUserCookieMappingItem& l, const TUserCookieMappingItem& r) {
- return GetKey(l) < GetKey(r);
- }
-};
-
-struct TConsumerCookieCmp {
- static TConsumerCookieKey GetKey(const TConsumerCookieMappingItem& item);
-
- static bool Compare(const TConsumerCookieKey& l, const TConsumerCookieMappingItem& r) {
- return l < GetKey(r);
- }
-
- static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieKey& r) {
- return GetKey(l) < r;
- }
-
- static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieMappingItem& r) {
- return GetKey(l) < GetKey(r);
- }
-};
-
-struct TUserCookieMappingItem : public TRbTreeItem<TUserCookieMappingItem, TUserCookieCmp> {
-};
-
-struct TConsumerCookieMappingItem : public TRbTreeItem<TConsumerCookieMappingItem, TConsumerCookieCmp> {
-};
-
-bool IsRetryable(const grpc::Status& status) {
- switch (status.error_code()) {
- case grpc::OK:
- case grpc::CANCELLED:
- case grpc::INVALID_ARGUMENT:
- case grpc::NOT_FOUND:
- case grpc::ALREADY_EXISTS:
- case grpc::PERMISSION_DENIED:
- case grpc::UNAUTHENTICATED:
- case grpc::FAILED_PRECONDITION:
- case grpc::ABORTED:
- case grpc::OUT_OF_RANGE:
- case grpc::UNIMPLEMENTED:
- return false;
-
- case grpc::UNKNOWN:
- case grpc::DEADLINE_EXCEEDED:
- case grpc::RESOURCE_EXHAUSTED:
- case grpc::INTERNAL:
- case grpc::UNAVAILABLE:
- case grpc::DATA_LOSS:
- case grpc::DO_NOT_USE:
- return true;
- }
-}
-
-} // namespace
-
-struct TMultiClusterConsumer::TCookieMappingItem : public TUserCookieMappingItem, public TConsumerCookieMappingItem {
- TCookieMappingItem(size_t subconsumer, ui64 consumerCookie, ui64 userCookie)
- : Subconsumer(subconsumer)
- , ConsumerCookie(consumerCookie)
- , UserCookie(userCookie)
- {
- }
-
- size_t Subconsumer = 0;
- ui64 ConsumerCookie = 0;
- ui64 UserCookie = 0;
-};
-
-namespace {
-
-ui64 TUserCookieCmp::GetKey(const TUserCookieMappingItem& item) {
- return static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item).UserCookie;
-}
-
-TConsumerCookieKey TConsumerCookieCmp::GetKey(const TConsumerCookieMappingItem& item) {
- const auto& src = static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item);
- return { src.Subconsumer, src.ConsumerCookie };
-}
-
-} // namespace
-
-struct TMultiClusterConsumer::TCookieMapping {
- using TUserCookieTree = TRbTree<TUserCookieMappingItem, TUserCookieCmp>;
- using TConsumerCookieTree = TRbTree<TConsumerCookieMappingItem, TConsumerCookieCmp>;
-
- struct TConsumerCookieDestroy : public TConsumerCookieTree::TDestroy {
- void operator()(TConsumerCookieMappingItem& item) const {
- TDestroy::operator()(item); // Remove from tree
- delete static_cast<TCookieMappingItem*>(&item);
- }
- };
-
- ~TCookieMapping() {
- UserCookieTree.ForEachNoOrder(TUserCookieTree::TDestroy());
- UserCookieTree.Init();
-
- ConsumerCookieTree.ForEachNoOrder(TConsumerCookieDestroy());
- ConsumerCookieTree.Init();
- }
-
- ui64 AddMapping(size_t subconsumer, ui64 cookie) { // Returns user cookie
- auto* newItem = new TCookieMappingItem(subconsumer, cookie, NextCookie++);
- UserCookieTree.Insert(newItem);
- ConsumerCookieTree.Insert(newItem);
- return newItem->UserCookie;
- }
-
- TCookieMappingItem* FindMapping(ui64 userCookie) {
- return DownCast(UserCookieTree.Find(userCookie));
- }
-
- TCookieMappingItem* FindMapping(size_t subconsumer, ui64 cookie) {
- return DownCast(ConsumerCookieTree.Find(TConsumerCookieKey(subconsumer, cookie)));
- }
-
- void RemoveUserCookieMapping(TCookieMappingItem* item) {
- Y_ASSERT(item);
- static_cast<TUserCookieMappingItem*>(item)->UnLink();
- }
-
- void RemoveMapping(TCookieMappingItem* item) {
- Y_ASSERT(item);
- Y_ASSERT(!static_cast<TUserCookieMappingItem*>(item)->ParentTree());
- static_cast<TUserCookieMappingItem*>(item)->UnLink(); // Just in case
- static_cast<TConsumerCookieMappingItem*>(item)->UnLink();
- delete item;
- }
-
-private:
- template <class T>
- static TCookieMappingItem* DownCast(T* item) {
- if (item) {
- return static_cast<TCookieMappingItem*>(item);
- } else {
- return nullptr;
- }
- }
-
- TUserCookieTree UserCookieTree;
- TConsumerCookieTree ConsumerCookieTree;
- ui64 NextCookie = 1;
-};
-
-TMultiClusterConsumer::TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger)
- : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , Logger(std::move(logger))
- , SessionId(CreateGuidAsString())
- , CookieMapping(MakeHolder<TCookieMapping>())
-{
- PatchSettings();
-}
-
-TMultiClusterConsumer::~TMultiClusterConsumer() {
- Destroy();
-}
-
-void TMultiClusterConsumer::PatchSettings() {
- if (!Settings.ReconnectOnFailure) {
- WARN_LOG("Ignoring ReconnectOnFailure=false option for multicluster consumer", "", SessionId);
- Settings.ReconnectOnFailure = true;
- }
- if (Settings.MaxAttempts != std::numeric_limits<unsigned>::max()) {
- WARN_LOG("Ignoring MaxAttempts option for multicluster consumer", "", SessionId);
- Settings.MaxAttempts = std::numeric_limits<unsigned>::max();
- }
-}
-
-NThreading::TFuture<TConsumerCreateResponse> TMultiClusterConsumer::Start(TInstant deadline) noexcept {
- Y_VERIFY(State == EState::Created);
- StartClusterDiscovery(deadline);
- return StartPromise.GetFuture();
-}
-
-NThreading::TFuture<TError> TMultiClusterConsumer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TMultiClusterConsumer::ScheduleClusterDiscoveryRetry(TInstant deadline) {
- const TInstant now = TInstant::Now();
- if (ClusterDiscoveryAttemptsDone >= Settings.MaxAttempts || now >= deadline) {
- Destroy(TStringBuilder() << "Failed " << ClusterDiscoveryAttemptsDone << " cluster discovery attempts", NErrorCode::CREATE_TIMEOUT);
- return;
- }
-
- if (!IsRetryable(ClusterDiscoverResult->first)) {
- Destroy(TStringBuilder() << "Cluster discovery failed: " << static_cast<int>(ClusterDiscoverResult->first.error_code()));
- return;
- }
-
- TDuration delay = Min(Settings.MaxReconnectionDelay, ClusterDiscoveryAttemptsDone * Settings.ReconnectionDelay, deadline - now);
- std::weak_ptr<TMultiClusterConsumer> weakRef = shared_from_this();
- PQLib->GetScheduler().Schedule(delay, this, [weakRef, deadline]() {
- if (auto self = weakRef.lock()) {
- self->StartClusterDiscovery(deadline);
- }
- });
-}
-
-void TMultiClusterConsumer::StartClusterDiscovery(TInstant deadline) {
- if (State == EState::Dead) {
- return;
- }
-
- State = EState::WaitingClusterDiscovery;
- ++ClusterDiscoveryAttemptsDone;
- DEBUG_LOG("Starting cluster discovery", "", SessionId);
-
- auto discoverer = MakeIntrusive<TConsumerChannelOverCdsImpl>(Settings, PQLib.Get(), Logger);
- discoverer->Start();
- ClusterDiscoverResult = discoverer->GetResultPtr();
- ClusterDiscoverer = std::move(discoverer);
-
- std::weak_ptr<TMultiClusterConsumer> self = shared_from_this();
- auto handler = [self, deadline](const auto&) {
- if (auto selfShared = self.lock()) {
- selfShared->OnClusterDiscoveryDone(deadline);
- }
- };
- PQLib->Subscribe(ClusterDiscoverer->GetChannel(), this, handler);
-}
-
-void TMultiClusterConsumer::OnClusterDiscoveryDone(TInstant deadline) {
- if (State != EState::WaitingClusterDiscovery) {
- return;
- }
-
- ClusterDiscoverer = nullptr; // delete
- if (!ClusterDiscoverResult->first.ok()) {
- INFO_LOG("Failed to discover clusters. Grpc error: " << static_cast<int>(ClusterDiscoverResult->first.error_code()), "", SessionId);
- ScheduleClusterDiscoveryRetry(deadline); // Destroys if we shouldn't retry.
- return;
- }
-
- if (static_cast<size_t>(ClusterDiscoverResult->second.read_sessions_clusters_size()) != Settings.Topics.size()) {
- Destroy("Got unexpected cluster discovery result");
- return;
- }
-
- if (deadline != TInstant::Max()) {
- const TInstant now = TInstant::Now();
- if (now > deadline) {
- Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT);
- return;
- }
-
- std::weak_ptr<TMultiClusterConsumer> self = shared_from_this();
- auto onStartTimeout = [self] {
- if (auto selfShared = self.lock()) {
- selfShared->OnStartTimeout();
- }
- };
- StartDeadlineCallback =
- PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout);
- }
-
- State = EState::StartingSubconsumers;
- DEBUG_LOG("Starting subconsumers", "", SessionId);
-
- // Group topics by clusters.
- THashMap<TString, TVector<TString>> clusterToTopics;
- for (size_t topicIndex = 0, topicsCount = Settings.Topics.size(); topicIndex < topicsCount; ++topicIndex) {
- const Ydb::PersQueue::ClusterDiscovery::ReadSessionClusters& clusters = ClusterDiscoverResult->second.read_sessions_clusters(topicIndex);
- const TString& topic = Settings.Topics[topicIndex];
- for (const Ydb::PersQueue::ClusterDiscovery::ClusterInfo& clusterInfo : clusters.clusters()) {
- clusterToTopics[clusterInfo.endpoint()].push_back(topic);
- }
- }
-
- if (clusterToTopics.empty()) {
- Destroy("Got empty endpoint set from cluster discovery");
- return;
- }
-
- // Start consumer on every cluster with its topics set.
- for (auto&& [clusterEndpoint, topics] : clusterToTopics) {
- // Settings.
- TConsumerSettings settings = Settings;
- settings.Server = ApplyClusterEndpoint(settings.Server, clusterEndpoint);
- settings.ReadFromAllClusterSources = false;
- settings.Topics = std::move(topics);
- settings.Unpack = false; // Unpack is being done in upper level decompressing consumer.
- settings.MaxMemoryUsage = Max(settings.MaxMemoryUsage / clusterToTopics.size(), static_cast<size_t>(1));
- if (settings.MaxUncommittedSize > 0) { // Limit is enabled.
- settings.MaxUncommittedSize = Max(settings.MaxUncommittedSize / clusterToTopics.size(), static_cast<size_t>(1));
- }
- if (settings.MaxUncommittedCount > 0) { // Limit is enabled.
- settings.MaxUncommittedCount = Max(settings.MaxUncommittedCount / clusterToTopics.size(), static_cast<size_t>(1));
- }
- const size_t subconsumerIndex = Subconsumers.size();
-
- // Create subconsumer.
- Subconsumers.push_back(PQLib->CreateRawRetryingConsumer(settings, DestroyEventRef, Logger));
-
- // Subscribe on start.
- std::weak_ptr<TMultiClusterConsumer> self = shared_from_this();
- auto handler = [self, subconsumerIndex](const auto&) {
- if (auto selfShared = self.lock()) {
- selfShared->OnSubconsumerStarted(subconsumerIndex);
- }
- };
- PQLib->Subscribe(Subconsumers.back().StartFuture, this, handler);
- }
-}
-
-void TMultiClusterConsumer::OnStartTimeout() {
- if (State == EState::Dead || State == EState::Working) {
- return;
- }
-
- StartDeadlineCallback = nullptr;
- Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT);
-}
-
-
-void TMultiClusterConsumer::OnSubconsumerStarted(size_t subconsumerIndex) {
- if (State == EState::Dead) {
- return;
- }
-
- auto& subconsumerInfo = Subconsumers[subconsumerIndex];
- const auto& result = subconsumerInfo.StartFuture.GetValue();
- if (result.Response.HasError()) {
- WARN_LOG("Got error on starting subconsumer: " << result.Response.GetError(), "", SessionId);
- Destroy(result.Response.GetError());
- return;
- }
-
- // Process start response
- subconsumerInfo.StartResponseWasGot = true;
- INFO_LOG("Subconsumer " << subconsumerIndex << " started session with id " << result.Response.GetInit().GetSessionId(), "", SessionId);
-
- // Move to new state
- if (State == EState::StartingSubconsumers) {
- State = EState::Working;
- if (StartDeadlineCallback) {
- StartDeadlineCallback->TryCancel();
- StartDeadlineCallback = nullptr;
- }
-
- {
- TReadResponse resp;
- resp.MutableInit()->SetSessionId(SessionId);
- StartPromise.SetValue(TConsumerCreateResponse(std::move(resp)));
- }
- } else {
- RequestSubconsumers(); // Make requests for new subconsumer.
- }
-}
-
-void TMultiClusterConsumer::Commit(const TVector<ui64>& cookies) noexcept {
- if (State != EState::Working) {
- Destroy("Requesting commit, but consumer is not in working state", NErrorCode::BAD_REQUEST);
- return;
- }
-
- TVector<TVector<ui64>> subconsumersCookies(Subconsumers.size());
- for (ui64 userCookie : cookies) {
- TCookieMappingItem* mapping = CookieMapping->FindMapping(userCookie);
- if (!mapping) {
- Destroy(TStringBuilder() << "Wrong cookie " << userCookie, NErrorCode::WRONG_COOKIE);
- return;
- }
- Y_ASSERT(mapping->Subconsumer < Subconsumers.size());
- subconsumersCookies[mapping->Subconsumer].push_back(mapping->ConsumerCookie);
- CookieMapping->RemoveUserCookieMapping(mapping); // Avoid double commit. This will ensure error on second commit with the same cookie.
- }
-
- for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) {
- const TVector<ui64>& subconsumerCommitRequest = subconsumersCookies[subconsumerIndex];
- if (!subconsumerCommitRequest.empty()) {
- Subconsumers[subconsumerIndex].Consumer->Commit(subconsumerCommitRequest);
- }
- }
-}
-
-void TMultiClusterConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept {
- if (State != EState::Working) {
- promise.SetValue(MakeResponse(MakeError("Requesting next message, but consumer is not in working state", NErrorCode::BAD_REQUEST)));
- return;
- }
-
- Requests.push_back(promise);
- CheckReadyResponses();
- RequestSubconsumers();
-}
-
-void TMultiClusterConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- if (State != EState::Working) {
- Destroy("Requesting partition status, but consumer is not in working state", NErrorCode::BAD_REQUEST);
- return;
- }
-
- const auto subconsumerIdIt = OldTopicName2Subconsumer.find(topic);
- if (subconsumerIdIt != OldTopicName2Subconsumer.end()) {
- Y_ASSERT(subconsumerIdIt->second < Subconsumers.size());
- Subconsumers[subconsumerIdIt->second].Consumer->RequestPartitionStatus(topic, partition, generation);
- } else {
- WARN_LOG("Requested partition status for topic \"" << topic << "\" (partition " << partition << ", generation " << generation << "), but there is no such lock session. Ignoring request", "", SessionId);
- }
-}
-
-void TMultiClusterConsumer::CheckReadyResponses() {
- const size_t prevCurrentSubconsumer = CurrentSubconsumer;
-
- do {
- if (!ProcessReadyResponses(CurrentSubconsumer)) {
- break;
- }
-
- // Next subconsumer in round robin way.
- ++CurrentSubconsumer;
- if (CurrentSubconsumer == Subconsumers.size()) {
- CurrentSubconsumer = 0;
- }
- } while (CurrentSubconsumer != prevCurrentSubconsumer && !Requests.empty());
-}
-
-bool TMultiClusterConsumer::ProcessReadyResponses(size_t subconsumerIndex) {
- if (Subconsumers.empty()) {
- Y_VERIFY(State == EState::Dead);
- return false;
- }
- Y_VERIFY(subconsumerIndex < Subconsumers.size());
- TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex];
- while (!consumerInfo.PendingRequests.empty() && !Requests.empty() && consumerInfo.PendingRequests.front().HasValue()) {
- if (!TranslateConsumerMessage(Requests.front(), consumerInfo.PendingRequests.front().ExtractValue(), subconsumerIndex)) {
- return false;
- }
- Requests.pop_front();
- consumerInfo.PendingRequests.pop_front();
- }
- return true;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- switch (subconsumerResponse.Type) {
- case EMT_LOCK:
- return TranslateConsumerMessageLock(promise, std::move(subconsumerResponse), subconsumerIndex);
- case EMT_RELEASE:
- return TranslateConsumerMessageRelease(promise, std::move(subconsumerResponse), subconsumerIndex);
- case EMT_DATA:
- return TranslateConsumerMessageData(promise, std::move(subconsumerResponse), subconsumerIndex);
- case EMT_ERROR:
- return TranslateConsumerMessageError(promise, std::move(subconsumerResponse), subconsumerIndex);
- case EMT_STATUS:
- return TranslateConsumerMessageStatus(promise, std::move(subconsumerResponse), subconsumerIndex);
- case EMT_COMMIT:
- return TranslateConsumerMessageCommit(promise, std::move(subconsumerResponse), subconsumerIndex);
- }
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- auto* data = subconsumerResponse.Response.MutableData();
- data->SetCookie(CookieMapping->AddMapping(subconsumerIndex, data->GetCookie()));
- promise.SetValue(std::move(subconsumerResponse));
- return true;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- for (ui64& consumerCookie : *subconsumerResponse.Response.MutableCommit()->MutableCookie()) {
- auto* mapping = CookieMapping->FindMapping(subconsumerIndex, consumerCookie);
- if (!mapping) {
- Destroy(TStringBuilder() << "Received unknown cookie " << consumerCookie << " commit");
- return false;
- }
- Y_VERIFY(mapping);
- consumerCookie = mapping->UserCookie;
- CookieMapping->RemoveMapping(mapping); // Invalidates mapping
- }
- promise.SetValue(std::move(subconsumerResponse));
- return true;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- OldTopicName2Subconsumer[subconsumerResponse.Response.GetLock().GetTopic()] = subconsumerIndex;
- promise.SetValue(std::move(subconsumerResponse));
- return true;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- Y_UNUSED(subconsumerIndex);
- promise.SetValue(std::move(subconsumerResponse));
- return true;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- WARN_LOG("Got error from subconsumer " << subconsumerIndex << ": " << subconsumerResponse.Response.GetError(), "", SessionId);
- Destroy(subconsumerResponse.Response.GetError());
- Y_UNUSED(promise);
- return false;
-}
-
-bool TMultiClusterConsumer::TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) {
- Y_UNUSED(subconsumerIndex);
- promise.SetValue(std::move(subconsumerResponse));
- return true;
-}
-
-void TMultiClusterConsumer::RequestSubconsumers() {
- const size_t inflight = Requests.size();
- if (!inflight) {
- return;
- }
- TMaybe<std::weak_ptr<TMultiClusterConsumer>> maybeSelf;
- for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) {
- TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex];
- if (!consumerInfo.StartResponseWasGot) {
- continue; // Consumer hasn't started yet.
- }
- while (consumerInfo.PendingRequests.size() < inflight) {
- if (!maybeSelf) {
- maybeSelf.ConstructInPlace(shared_from_this()); // Don't construct it if we don't need it.
- }
- consumerInfo.PendingRequests.push_back(consumerInfo.Consumer->GetNextMessage());
- PQLib->Subscribe(consumerInfo.PendingRequests.back(),
- this,
- [self = *maybeSelf, subconsumerIndex](const auto&) {
- if (auto selfShared = self.lock()) {
- selfShared->ProcessReadyResponses(subconsumerIndex);
- }
- });
- }
- }
-}
-
-void TMultiClusterConsumer::Destroy(const TError& description) {
- if (State == EState::Dead) {
- return;
- }
- State = EState::Dead;
-
- WARN_LOG("Destroying consumer with error description: " << description, "", SessionId);
-
- StartPromise.TrySetValue(MakeCreateResponse(description));
-
- if (StartDeadlineCallback) {
- StartDeadlineCallback->TryCancel();
- }
-
- for (auto& reqPromise : Requests) {
- reqPromise.SetValue(MakeResponse(description));
- }
- Requests.clear();
- Subconsumers.clear();
-
- IsDeadPromise.SetValue(description);
-
- DestroyPQLibRef();
-}
-
-void TMultiClusterConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) {
- Destroy(MakeError(description, code));
-}
-
-void TMultiClusterConsumer::Destroy() {
- Destroy(GetCancelReason());
-}
-
-void TMultiClusterConsumer::Cancel() {
- Destroy(GetCancelReason());
-}
-
-TError TMultiClusterConsumer::MakeError(const TString& description, NErrorCode::EErrorCode code) {
- TError error;
- error.SetDescription(description);
- error.SetCode(code);
- return error;
-}
-
-TConsumerCreateResponse TMultiClusterConsumer::MakeCreateResponse(const TError& description) {
- TReadResponse res;
- res.MutableError()->CopyFrom(description);
- return TConsumerCreateResponse(std::move(res));
-}
-
-TConsumerMessage TMultiClusterConsumer::MakeResponse(const TError& description) {
- TReadResponse res;
- res.MutableError()->CopyFrom(description);
- return TConsumerMessage(std::move(res));
-}
-
-TMultiClusterConsumer::TSubconsumerInfo::TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer)
- : Consumer(std::move(consumer))
- , StartFuture(Consumer->Start())
- , DeadFuture(Consumer->IsDead())
-{
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h
deleted file mode 100644
index ab8a338fbe..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h
+++ /dev/null
@@ -1,93 +0,0 @@
-#pragma once
-#include "channel_p.h"
-#include "scheduler.h"
-#include "iconsumer_p.h"
-#include "internals.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class TMultiClusterConsumer: public IConsumerImpl, public std::enable_shared_from_this<TMultiClusterConsumer> {
-public:
- struct TSubconsumerInfo;
- struct TCookieMappingItem;
- struct TCookieMapping;
-
-public:
- TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger);
- ~TMultiClusterConsumer();
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override;
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override;
- void Commit(const TVector<ui64>& cookies) noexcept override;
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override;
-
- void Cancel() override;
-
-private:
- void PatchSettings();
- void StartClusterDiscovery(TInstant deadline);
- void OnClusterDiscoveryDone(TInstant deadline);
- void OnStartTimeout();
- void OnSubconsumerStarted(size_t subconsumerIndex);
-
- void CheckReadyResponses();
- bool ProcessReadyResponses(size_t subconsumerIndex); // False if consumer has been destroyed.
- void RequestSubconsumers(); // Make requests inflight in every subconsumer equal to our superconsumer inflight.
-
- bool TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // False if consumer has been destroyed.
- bool TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex);
- bool TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex);
- bool TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex);
- bool TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex);
- bool TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // Calls destroy on consumer
- bool TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex);
-
- void Destroy(const TError& description);
- void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR);
- void Destroy();
-
- static TError MakeError(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR);
- static TConsumerCreateResponse MakeCreateResponse(const TError& description);
- static TConsumerMessage MakeResponse(const TError& description);
-
- void ScheduleClusterDiscoveryRetry(TInstant deadline);
-
-private:
- TConsumerSettings Settings;
- TIntrusivePtr<ILogger> Logger;
- TString SessionId;
-
- NThreading::TPromise<TConsumerCreateResponse> StartPromise = NThreading::NewPromise<TConsumerCreateResponse>();
- NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>();
-
- enum class EState {
- Created, // Before Start() was created.
- WaitingClusterDiscovery, // After Start() was called and before we received CDS response.
- StartingSubconsumers, // After we received CDS response and before consumer actually started.
- Working, // When one or more consumers had started.
- Dead,
- };
- EState State = EState::Created;
-
- TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback;
- TChannelImplPtr ClusterDiscoverer;
- TConsumerChannelOverCdsImpl::TResultPtr ClusterDiscoverResult;
- size_t ClusterDiscoveryAttemptsDone = 0;
-
- std::vector<TSubconsumerInfo> Subconsumers;
- size_t CurrentSubconsumer = 0; // Index of current subconsumer to take responses from
-
- THolder<TCookieMapping> CookieMapping;
- THashMap<TString, size_t> OldTopicName2Subconsumer; // Used in lock session for requesting partition status.
-
- std::deque<NThreading::TPromise<TConsumerMessage>> Requests;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp
deleted file mode 100644
index a29a18cbaf..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp
+++ /dev/null
@@ -1,227 +0,0 @@
-#include "multicluster_consumer.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/event.h>
-
-namespace NPersQueue {
-
-Y_UNIT_TEST_SUITE(TMultiClusterConsumerTest) {
- TConsumerSettings MakeFakeConsumerSettings() {
- TConsumerSettings settings;
- settings.Topics.push_back("topic");
- settings.ReadFromAllClusterSources = true;
- settings.ReadMirroredPartitions = false;
- settings.MaxAttempts = 1; // Consumer should fix this setting. Check it.
- return settings;
- }
-
- TConsumerSettings MakeConsumerSettings(const SDKTestSetup& setup) {
- TConsumerSettings settings = setup.GetConsumerSettings();
- settings.ReadFromAllClusterSources = true;
- settings.ReadMirroredPartitions = false;
- settings.MaxAttempts = 1; // Consumer should fix this setting. Check it.
- return settings;
- }
-
- Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) {
- // Test that consumer doesn't hang on till shutdown
- TPQLib lib;
- lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG));
- }
-
- Y_UNIT_TEST(StartedConsumerCanBeDestructed) {
- // Test that consumer doesn't hang on till shutdown
- TPQLib lib;
- auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG));
- auto isDead = consumer->IsDead();
- consumer->Start();
- consumer = nullptr;
- isDead.GetValueSync();
- }
-
- Y_UNIT_TEST(CommitRightAfterStart) {
- SDKTestSetup setup("InfiniteStart");
- TPQLib lib;
- auto settings = MakeConsumerSettings(setup);
- settings.Topics[0] = "unknown_topic";
- auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG));
- consumer->Start();
- auto isDead = consumer->IsDead();
- consumer->Commit({42});
- if (GrpcV1EnabledByDefault()) {
- UNIT_ASSERT_EQUAL_C(isDead.GetValueSync().GetCode(), NErrorCode::WRONG_COOKIE, isDead.GetValueSync());
- } else {
- UNIT_ASSERT_STRINGS_EQUAL_C(isDead.GetValueSync().GetDescription(), "Requesting commit, but consumer is not in working state", isDead.GetValueSync());
- }
- }
-
- Y_UNIT_TEST(GetNextMessageRightAfterStart) {
- if (GrpcV1EnabledByDefault()) {
- return;
- }
- SDKTestSetup setup("InfiniteStart");
- TPQLib lib;
- auto settings = MakeConsumerSettings(setup);
- settings.Topics[0] = "unknown_topic";
- auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG));
- consumer->Start();
- auto nextMessage = consumer->GetNextMessage();
- UNIT_ASSERT_STRINGS_EQUAL(nextMessage.GetValueSync().Response.GetError().GetDescription(), "Requesting next message, but consumer is not in working state");
- }
-
- Y_UNIT_TEST(RequestPartitionStatusRightAfterStart) {
- if (GrpcV1EnabledByDefault()) {
- return;
- }
- SDKTestSetup setup("InfiniteStart");
- TPQLib lib;
- auto settings = MakeConsumerSettings(setup);
- settings.Topics[0] = "unknown_topic";
- auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG));
- consumer->Start();
- auto isDead = consumer->IsDead();
- consumer->RequestPartitionStatus("topic", 42, 42);
- UNIT_ASSERT_STRINGS_EQUAL(isDead.GetValueSync().GetDescription(), "Requesting partition status, but consumer is not in working state");
- }
-
- Y_UNIT_TEST(StartedConsumerCantBeStartedAgain) {
- TPQLib lib;
- auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG));
- auto startFuture = consumer->Start();
- auto startFuture2 = consumer->Start();
- UNIT_ASSERT(startFuture2.HasValue());
- UNIT_ASSERT_STRINGS_EQUAL(startFuture2.GetValueSync().Response.GetError().GetDescription(), "Start was already called");
- }
-
- Y_UNIT_TEST(StartWithFailedCds) {
- TPQLib lib;
- auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG));
- auto startFuture = consumer->Start(TDuration::MilliSeconds(10));
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT_STRING_CONTAINS(startFuture.GetValueSync().Response.GetError().GetDescription(), " cluster discovery attempts");
- }
- auto isDead = consumer->IsDead();
- isDead.Wait();
- }
-
- Y_UNIT_TEST(StartDeadline) {
- if (GrpcV1EnabledByDefault()) {
- return;
- }
- TPQLib lib;
- TConsumerSettings settings = MakeFakeConsumerSettings();
- auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG));
- auto startFuture = consumer->Start(TDuration::MilliSeconds(100));
- UNIT_ASSERT_EQUAL_C(startFuture.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startFuture.GetValueSync().Response);
- }
-
- Y_UNIT_TEST(StartDeadlineAfterCds) {
- SDKTestSetup setup("StartDeadlineAfterCds");
- TPQLib lib;
- auto settings = MakeConsumerSettings(setup);
- settings.Topics[0] = "unknown_topic";
- auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG));
- auto startFuture = consumer->Start(TDuration::MilliSeconds(1));
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "Start timeout");
- }
- auto isDead = consumer->IsDead();
- isDead.Wait();
- }
-
- void WorksWithSingleDc(bool oneDcIsUnavailable) {
- SDKTestSetup setup("WorksWithSingleDc", false);
- if (!oneDcIsUnavailable) {
- setup.SetSingleDataCenter(); // By default one Dc is fake in test setup. This call overrides it.
- }
- setup.Start();
- auto consumer = setup.StartConsumer(MakeConsumerSettings(setup));
- setup.WriteToTopic({"msg1", "msg2"});
- setup.ReadFromTopic({{"msg1", "msg2"}}, true, consumer.Get());
- }
-
- Y_UNIT_TEST(WorksWithSingleDc) {
- WorksWithSingleDc(false);
- }
-
- Y_UNIT_TEST(WorksWithSingleDcWhileSecondDcIsUnavailable) {
- WorksWithSingleDc(true);
- }
-
- Y_UNIT_TEST(FailsOnCommittingWrongCookie) {
- SDKTestSetup setup("FailsOnCommittingWrongCookie");
- auto consumer = setup.StartConsumer(MakeConsumerSettings(setup));
- setup.WriteToTopic({"msg1", "msg2"});
- auto nextMessage = consumer->GetNextMessage();
- UNIT_ASSERT_EQUAL(nextMessage.GetValueSync().Type, EMT_DATA);
- auto isDead = consumer->IsDead();
- consumer->Commit({nextMessage.GetValueSync().Response.GetData().GetCookie() + 1});
- UNIT_ASSERT_STRING_CONTAINS(isDead.GetValueSync().GetDescription(), "Wrong cookie");
- }
-
- // Test that consumer is properly connected with decompressing consumer.
- void Unpacks(bool unpack) {
- SDKTestSetup setup(TStringBuilder() << "Unpacks(" << (unpack ? "true)" : "false)"));
- setup.WriteToTopic({"msg"});
- auto settings = MakeConsumerSettings(setup);
- settings.Unpack = unpack;
- auto consumer = setup.StartConsumer(settings);
- auto readResponse = consumer->GetNextMessage().GetValueSync();
- UNIT_ASSERT_C(!readResponse.Response.HasError(), readResponse.Response);
- UNIT_ASSERT_C(readResponse.Response.HasData(), readResponse.Response);
- if (unpack) {
- UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::RAW, readResponse.Response);
- UNIT_ASSERT_STRINGS_EQUAL(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg");
- } else {
- UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::GZIP, readResponse.Response);
- }
- }
-
- Y_UNIT_TEST(UnpacksIfNeeded) {
- Unpacks(true);
- }
-
- Y_UNIT_TEST(DoesNotUnpackIfNeeded) {
- Unpacks(false);
- }
-
- Y_UNIT_TEST(WorksWithManyDc) {
- SDKTestSetup setup1("WorksWithManyDc1", false);
- SDKTestSetup setup2("WorksWithManyDc2");
- SDKTestSetup setup3("WorksWithManyDc3", false);
- setup1.AddDataCenter("dc2", setup2);
- setup1.AddDataCenter("dc3", setup3);
- setup3.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::Seconds(1));
- setup3.Start();
- setup1.Start();
-
- // Check that consumer reads from all sources.
- setup1.WriteToTopic({"msg1", "msg2"});
- setup2.WriteToTopic({"msg3", "msg4"});
- setup3.WriteToTopic({"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"});
- auto consumer = setup1.StartConsumer(MakeConsumerSettings(setup1));
- setup1.ReadFromTopic({{"msg1", "msg2"}, {"msg3", "msg4"}, {"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"}}, true, consumer.Get());
-
- // Turn Dc off and check that consumer works.
- setup1.GetLog() << TLOG_INFO << "Turn off DataCenter dc3";
- setup3.ShutdownGRpc();
- setup1.WriteToTopic({"a", "b"});
- setup2.WriteToTopic({"x", "y", "z"});
- setup1.ReadFromTopic({{"a", "b"}, {"x", "y", "z"}}, true, consumer.Get());
-
- // Turn Dc on again and check work again.
- setup1.GetLog() << TLOG_INFO << "Turn on DataCenter dc3";
- setup3.EnableGRpc();
- setup1.WriteToTopic({"1", "2"});
- setup3.WriteToTopic({"42"});
- setup1.ReadFromTopic({{"1", "2"}, {"42"}}, true, consumer.Get());
- }
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp
deleted file mode 100644
index 6f63b0ab4a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp
+++ /dev/null
@@ -1,423 +0,0 @@
-#include "multicluster_producer.h"
-#include "persqueue_p.h"
-
-#include <util/string/builder.h>
-
-#include <algorithm>
-
-namespace NPersQueue {
-
-TWeightIndex::TWeightIndex(size_t count)
- : Weights(count)
- , Index(count)
- , IsEnabled(count)
- , EnabledCnt(0)
-{
- Y_VERIFY(count > 0);
-}
-
-void TWeightIndex::SetWeight(size_t i, unsigned weight) {
- const unsigned oldWeight = Weights[i];
- const int diff = static_cast<int>(weight) - static_cast<int>(oldWeight);
- if (diff != 0) {
- Weights[i] = weight;
- if (IsEnabled[i]) {
- UpdateIndex(i, diff);
- }
- }
-}
-
-void TWeightIndex::Enable(size_t i) {
- if (!IsEnabled[i]) {
- ++EnabledCnt;
- IsEnabled[i] = true;
- UpdateIndex(i, static_cast<int>(Weights[i]));
- }
-}
-
-void TWeightIndex::Disable(size_t i) {
- if (IsEnabled[i]) {
- --EnabledCnt;
- IsEnabled[i] = false;
- UpdateIndex(i, -static_cast<int>(Weights[i]));
- }
-}
-
-size_t TWeightIndex::Choose(unsigned randomNumber) const {
- Y_VERIFY(randomNumber < WeightsSum());
- Y_VERIFY(!Index.empty());
- const auto choice = std::upper_bound(Index.begin(), Index.end(), randomNumber);
- Y_VERIFY(choice != Index.end());
- return choice - Index.begin();
-}
-
-void TWeightIndex::UpdateIndex(size_t i, int weightDiff) {
- for (size_t j = i; j < Index.size(); ++j) {
- Index[j] += weightDiff;
- }
-}
-
-static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) {
- TWriteResponse res;
- res.MutableError()->SetDescription(description);
- res.MutableError()->SetCode(NErrorCode::ERROR);
- return TProducerCommitResponse{seqNo, data, std::move(res)};
-}
-
-TMultiClusterProducer::TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger)
- : IProducerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , DeadPromise(NThreading::NewPromise<TError>())
- , StartPromise(NThreading::NewPromise<TProducerCreateResponse>())
- , State(EState::Created)
- , Logger(std::move(logger))
- , WeightIndex(Settings.ServerWeights.size())
- , FallbackWeightIndex(Settings.ServerWeights.size())
- , Subproducers(Settings.ServerWeights.size())
- , FallbackSubproducers(Settings.ServerWeights.size())
-{
- if (Settings.ServerWeights.size() <= 1) {
- ythrow yexception() << "MultiCluster producer is working with several servers. You'd better to create retrying producer in this case";
- }
- if (Settings.MinimumWorkingDcsCount == 0) {
- ythrow yexception() << "MinimumWorkingDcsCount can't be zero. We want that you have at least something working!";
- }
- if (Settings.SourceIdPrefix.empty()) {
- ythrow yexception() << "SourceIdPrefix must be nonempty";
- }
-
- bool reconnectWarningPrinted = false;
- for (TMultiClusterProducerSettings::TServerWeight& w : Settings.ServerWeights) {
- if (!w.Weight) {
- ythrow yexception() << "Server weight must be greater than zero";
- }
- if (!w.ProducerSettings.ReconnectOnFailure) {
- w.ProducerSettings.ReconnectOnFailure = true;
- if (!reconnectWarningPrinted) {
- reconnectWarningPrinted = true;
- WARN_LOG("ReconnectOnFailure setting is off. Turning it on for multicluster producer.", Settings.SourceIdPrefix, "");
- }
- ythrow yexception() << "You must enable ReconnectOnFailure setting";
- }
- if (w.ProducerSettings.MaxAttempts == std::numeric_limits<unsigned>::max()) {
- ythrow yexception() << "MaxAttempts must be a finite number";
- }
- }
-
- for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) {
- WeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight);
- FallbackWeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight);
- }
-}
-
-TMultiClusterProducer::~TMultiClusterProducer() {
- Destroy("Destructor called");
-}
-
-void TMultiClusterProducer::StartSubproducer(TInstant deadline, size_t i, bool fallback) {
- std::weak_ptr<TMultiClusterProducer> self(shared_from_this());
- auto& sp = GetSubproducers(fallback)[i];
- sp.Producer = CreateSubproducer(Settings.ServerWeights[i].ProducerSettings, i, fallback);
- sp.StartFuture = sp.Producer->Start(deadline);
- sp.DeadFuture = sp.Producer->IsDead();
- PQLib->Subscribe(sp.StartFuture,
- this,
- [self, i, fallback](const auto&) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnProducerStarted(i, fallback);
- }
- });
- PQLib->Subscribe(sp.DeadFuture,
- this,
- [self, i, fallback](const auto&) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnProducerDead(i, fallback);
- }
- });
-}
-
-void TMultiClusterProducer::OnNeedProducerRestart(size_t i, bool fallback) {
- if (State == EState::Dead) {
- return;
- }
- const auto deadline = TInstant::Now() + Settings.ServerWeights[i].ProducerSettings.StartSessionTimeout;
- StartSubproducer(deadline, i, fallback);
-}
-
-void TMultiClusterProducer::ScheduleProducerRestart(size_t i, bool fallback) {
- std::weak_ptr<TMultiClusterProducer> self(shared_from_this());
- PQLib->GetScheduler().Schedule(Settings.ServerWeights[i].ProducerSettings.ReconnectionDelay,
- this,
- [self, i, fallback] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnNeedProducerRestart(i, fallback);
- }
- });
-}
-
-void TMultiClusterProducer::StartSubproducers(TInstant deadline, bool fallback) {
- for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) {
- StartSubproducer(deadline, i, fallback);
- }
-}
-
-NThreading::TFuture<TProducerCreateResponse> TMultiClusterProducer::Start(TInstant deadline) noexcept {
- Y_VERIFY(State == EState::Created);
- State = EState::Starting;
- StartSubproducers(deadline);
- StartSubproducers(deadline, true);
- return StartPromise.GetFuture();
-}
-
-void TMultiClusterProducer::OnProducerStarted(size_t i, bool fallback) {
- if (State == EState::Dead) {
- return;
- }
- TSubproducerInfo& info = GetSubproducers(fallback)[i];
- info.StartAnswered = true;
- DEBUG_LOG("Start subproducer response: " << info.StartFuture.GetValue().Response, Settings.SourceIdPrefix, "");
- if (info.StartFuture.GetValue().Response.HasError()) {
- ERR_LOG("Failed to start subproducer[" << i << "]: " << info.StartFuture.GetValue().Response.GetError().GetDescription(), Settings.SourceIdPrefix, "");
- if (State == EState::Starting && !fallback) {
- size_t startedCount = 0;
- for (const auto& p : Subproducers) {
- startedCount += p.StartAnswered;
- }
- if (startedCount == Subproducers.size() && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) {
- Destroy("Not enough subproducers started successfully");
- return;
- }
- }
-
- ScheduleProducerRestart(i, fallback);
- } else {
- GetWeightIndex(fallback).Enable(i);
- if (State == EState::Starting && !fallback && WeightIndex.EnabledCount() >= Settings.MinimumWorkingDcsCount) {
- State = EState::Working;
- TWriteResponse response;
- response.MutableInit();
- StartPromise.SetValue(TProducerCreateResponse(std::move(response)));
- }
- if (fallback) {
- ResendLastHopeQueue(i);
- }
- }
-}
-
-void TMultiClusterProducer::OnProducerDead(size_t i, bool fallback) {
- if (State == EState::Dead) {
- return;
- }
-
- WARN_LOG("Subproducer[" << i << "] is dead: " << GetSubproducers(fallback)[i].DeadFuture.GetValue().GetDescription(), Settings.SourceIdPrefix, "");
- GetWeightIndex(fallback).Disable(i);
- if (!fallback && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) {
- Destroy("Not enough subproducers is online");
- return;
- }
- ScheduleProducerRestart(i, fallback);
-}
-
-void TMultiClusterProducer::Destroy(const TString& description) {
- TError error;
- error.SetCode(NErrorCode::ERROR);
- error.SetDescription(description);
- Destroy(error);
-}
-
-void TMultiClusterProducer::DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites) {
- for (TWriteInfo& wi : pendingWrites) {
- TWriteResponse response;
- response.MutableError()->CopyFrom(error);
- wi.ResponsePromise.SetValue(TProducerCommitResponse(0, std::move(wi.Data), std::move(response)));
- }
- pendingWrites.Clear();
-}
-
-void TMultiClusterProducer::Destroy(const TError& error) {
- if (State == EState::Dead) {
- return;
- }
-
- DEBUG_LOG("Destroying multicluster producer. Reason: " << error, Settings.SourceIdPrefix, "");
- const EState prevState = State;
- State = EState::Dead;
- SubscribeDestroyed();
- if (prevState == EState::Starting) {
- TWriteResponse response;
- response.MutableError()->CopyFrom(error);
- StartPromise.SetValue(TProducerCreateResponse(std::move(response)));
- }
- for (size_t i = 0; i < Subproducers.size(); ++i) {
- Subproducers[i].Producer = nullptr;
- FallbackSubproducers[i].Producer = nullptr;
- DestroyWrites(error, FallbackSubproducers[i].LastHopePendingWrites);
- }
- DestroyWrites(error, PendingWrites);
- DeadPromise.SetValue(error);
-
- DestroyPQLibRef();
-}
-
-void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept {
- return promise.SetValue(MakeErrorCommitResponse("MultiCluster producer doesn't allow to write with explicitly specified seq no.", seqNo, data));
-}
-
-void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept {
- Y_VERIFY(data.IsEncoded());
-
- if (State != EState::Working) {
- TWriteResponse res;
- res.MutableError()->SetDescription(TStringBuilder() << "producer is " << (State == EState::Dead ? "dead" : "not ready") << ". Marker# PQLib01");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- promise.SetValue(TProducerCommitResponse{0, std::move(data), std::move(res)});
- return;
- }
-
- TWriteInfo* wi = new TWriteInfo();
- PendingWrites.PushBack(wi);
-
- const size_t i = WeightIndex.RandomChoose();
- wi->Data = std::move(data);
- wi->ResponsePromise = promise;
- DelegateWrite(wi, i);
-}
-
-void TMultiClusterProducer::DelegateWrite(TWriteInfo* wi, size_t i, bool fallback) {
- if (wi->History.empty() || wi->History.back() != i) {
- wi->History.push_back(i);
- }
- DEBUG_LOG("Delegate" << (fallback ? " fallback" : "") << " write to subproducer[" << i << "]", Settings.SourceIdPrefix, "");
- wi->ResponseFuture = GetSubproducers(fallback)[i].Producer->Write(wi->Data);
- wi->WaitingCallback = true;
-
- std::weak_ptr<TMultiClusterProducer> self(shared_from_this());
- PQLib->Subscribe(wi->ResponseFuture,
- this,
- [self, wi](const auto&) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnWriteResponse(wi);
- }
- });
-}
-
-void TMultiClusterProducer::ResendLastHopeQueue(size_t i) {
- TSubproducerInfo& info = FallbackSubproducers[i];
- if (!info.LastHopePendingWrites.Empty()) {
- for (TWriteInfo& wi : info.LastHopePendingWrites) {
- if (!wi.WaitingCallback) {
- DelegateWrite(&wi, i, true);
- }
- }
- }
-}
-
-void TMultiClusterProducer::OnWriteResponse(TWriteInfo* wi) {
- if (State == EState::Dead) {
- return;
- }
-
- Y_VERIFY(wi->WaitingCallback);
- Y_VERIFY(!wi->History.empty());
-
- DEBUG_LOG("Write response: " << wi->ResponseFuture.GetValue().Response, Settings.SourceIdPrefix, "");
- wi->WaitingCallback = false;
- if (wi->ResponseFuture.GetValue().Response.HasAck()) {
- wi->ResponsePromise.SetValue(wi->ResponseFuture.GetValue());
- wi->Unlink();
- delete wi;
- } else if (wi->ResponseFuture.GetValue().Response.HasError()) {
- // retry write to other dc
- if (wi->LastHope) {
- const size_t i = wi->History.back();
- if (FallbackWeightIndex.Enabled(i)) {
- DelegateWrite(wi, i, true);
- } // else we will retry when this producer will become online.
- return;
- }
- TWeightIndex index = FallbackWeightIndex;
- for (size_t i : wi->History) {
- index.Disable(i);
- }
-
- // choose new fallback dc
- size_t i;
- if (index.EnabledCount() == 0) {
- i = wi->History.back();
- } else {
- i = index.RandomChoose();
- }
-
- // define whether this dc is the last hope
- const bool lastHope = index.EnabledCount() <= 1;
- if (lastHope) {
- wi->LastHope = true;
- wi->Unlink();
- FallbackSubproducers[i].LastHopePendingWrites.PushBack(wi);
- }
-
- DEBUG_LOG("Retrying write to subproducer[" << i << "], lastHope: " << lastHope, Settings.SourceIdPrefix, "");
- DelegateWrite(wi, i, true);
- } else { // incorrect structure of response
- Y_VERIFY(false);
- }
-}
-
-NThreading::TFuture<TError> TMultiClusterProducer::IsDead() noexcept {
- return DeadPromise.GetFuture();
-}
-
-TString TMultiClusterProducer::MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId,
- size_t producerIndex, bool fallback) {
- TStringBuilder ret;
- ret << prefix << "-" << producerIndex;
- if (!srcSourceId.empty()) {
- ret << "-"sv << srcSourceId;
- }
- if (fallback) {
- ret << "-multicluster-fallback"sv;
- } else {
- ret << "-multicluster"sv;
- }
- return std::move(ret);
-}
-
-std::shared_ptr<IProducerImpl> TMultiClusterProducer::CreateSubproducer(const TProducerSettings& srcSettings,
- size_t index, bool fallback) {
- TProducerSettings settings = srcSettings;
- settings.SourceId = MakeSourceId(Settings.SourceIdPrefix, srcSettings.SourceId, index, fallback);
- return PQLib->CreateRawRetryingProducer(settings, DestroyEventRef, Logger);
-}
-
-void TMultiClusterProducer::SubscribeDestroyed() {
- NThreading::TPromise<void> promise = ProducersDestroyed;
- auto handler = [promise](const auto&) mutable {
- promise.SetValue();
- };
- std::vector<NThreading::TFuture<void>> futures;
- futures.reserve(Subproducers.size() * 2 + 1);
- futures.push_back(DestroyedPromise.GetFuture());
- for (size_t i = 0; i < Subproducers.size(); ++i) {
- if (Subproducers[i].Producer) {
- futures.push_back(Subproducers[i].Producer->Destroyed());
- }
- if (FallbackSubproducers[i].Producer) {
- futures.push_back(FallbackSubproducers[i].Producer->Destroyed());
- }
- }
- WaitExceptionOrAll(futures).Subscribe(handler);
-}
-
-NThreading::TFuture<void> TMultiClusterProducer::Destroyed() noexcept {
- return ProducersDestroyed.GetFuture();
-}
-
-void TMultiClusterProducer::Cancel() {
- Destroy(GetCancelReason());
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h
deleted file mode 100644
index 16fcfb6b11..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h
+++ /dev/null
@@ -1,147 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include "internals.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include "channel.h"
-#include "iproducer_p.h"
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/intrlist.h>
-#include <util/random/random.h>
-
-#include <deque>
-
-namespace NPersQueue {
-
-// Structure to perform choose by weights
-class TWeightIndex {
-public:
- // Creates zero-weighted disabled index
- explicit TWeightIndex(size_t count);
-
- void SetWeight(size_t i, unsigned weight);
-
- void Enable(size_t i);
- void Disable(size_t i);
-
- // Chooses by random number in interval [0, WeightsSum())
- size_t Choose(unsigned randomNumber) const;
-
- size_t RandomChoose() const {
- return Choose(RandomNumber<unsigned>(WeightsSum()));
- }
-
- size_t EnabledCount() const {
- return EnabledCnt;
- }
-
- bool Enabled(size_t i) const {
- return i < IsEnabled.size() ? IsEnabled[i] : false;
- }
-
- unsigned WeightsSum() const {
- return Index.back();
- }
-
-private:
- void UpdateIndex(size_t i, int weightDiff);
-
-private:
- std::vector<unsigned> Weights;
- std::vector<unsigned> Index; // structure for choosing. Index[i] is sum of enabled weights with index <= i.
- std::vector<bool> IsEnabled;
- size_t EnabledCnt;
-};
-
-class TMultiClusterProducer: public IProducerImpl, public std::enable_shared_from_this<TMultiClusterProducer> {
-public:
- TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger);
- ~TMultiClusterProducer();
-
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override;
-
- using IProducerImpl::Write;
-
- // Forbidden write
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override;
-
- // Allowed write
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override;
-
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- NThreading::TFuture<void> Destroyed() noexcept override;
-
- void Cancel() override;
-
-private:
- struct TWriteInfo: public TIntrusiveListItem<TWriteInfo> {
- NThreading::TPromise<TProducerCommitResponse> ResponsePromise;
- NThreading::TFuture<TProducerCommitResponse> ResponseFuture;
- bool WaitingCallback = false;
- std::vector<size_t> History; // Number of DCs with failed or pending writes
- TData Data;
- bool LastHope = false; // This DC is the last hope: retry till success.
- // If this flag is set, this write info is in the local subproducer queue.
- };
-
- struct TSubproducerInfo {
- size_t Number = std::numeric_limits<size_t>::max();
- std::shared_ptr<IProducerImpl> Producer;
- NThreading::TFuture<TProducerCreateResponse> StartFuture;
- NThreading::TFuture<TError> DeadFuture;
- TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> LastHopePendingWrites; // Local queue with writes only to this DC.
- bool StartAnswered = false;
- };
-
-private:
- std::shared_ptr<IProducerImpl> CreateSubproducer(const TProducerSettings& settings, size_t index, bool fallback = false);
-
- std::vector<TSubproducerInfo>& GetSubproducers(bool fallback = false) {
- return fallback ? FallbackSubproducers : Subproducers;
- }
-
- TWeightIndex& GetWeightIndex(bool fallback = false) {
- return fallback ? FallbackWeightIndex : WeightIndex;
- }
-
- static TString MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId,
- size_t producerIndex, bool fallback);
- void StartSubproducers(TInstant deadline, bool fallback = false);
- void StartSubproducer(TInstant deadline, size_t i, bool fallback = false);
- void OnProducerStarted(size_t i, bool fallback);
- void OnProducerDead(size_t i, bool fallback);
- void OnWriteResponse(TWriteInfo* wi);
- void ScheduleProducerRestart(size_t i, bool fallback);
- void OnNeedProducerRestart(size_t i, bool fallback); // scheduler action
- void Destroy(const TString& description);
- void Destroy(const TError& error);
- void DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites);
- void DelegateWrite(TWriteInfo* wi, size_t i, bool fallback = false);
- void ResendLastHopeQueue(size_t i);
- void SubscribeDestroyed();
-
-protected:
- TMultiClusterProducerSettings Settings;
- NThreading::TPromise<TError> DeadPromise;
- NThreading::TPromise<TProducerCreateResponse> StartPromise;
- enum class EState {
- Created,
- Starting,
- Working,
- Dead,
- };
- EState State;
- TIntrusivePtr<ILogger> Logger;
- TWeightIndex WeightIndex;
- TWeightIndex FallbackWeightIndex;
- std::vector<TSubproducerInfo> Subproducers;
- std::vector<TSubproducerInfo> FallbackSubproducers;
- TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> PendingWrites;
- NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>();
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp
deleted file mode 100644
index 6942681e6a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp
+++ /dev/null
@@ -1,360 +0,0 @@
-#include "multicluster_producer.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/event.h>
-
-#include <atomic>
-
-namespace NPersQueue {
-
-Y_UNIT_TEST_SUITE(TWeightIndexTest) {
- Y_UNIT_TEST(Enables) {
- TWeightIndex wi(3);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0);
-
- wi.SetWeight(0, 1);
- wi.SetWeight(1, 2);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0);
-
- wi.Disable(1);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0);
-
- wi.Enable(0);
- wi.Enable(0); // no changes
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1);
-
- wi.Enable(1);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 2);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 3);
-
- wi.Disable(1);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1);
-
- wi.Disable(1); // no changes
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1);
- }
-
- Y_UNIT_TEST(Chooses) {
- const size_t weightsCount = 3;
- TWeightIndex wi(weightsCount);
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0);
- unsigned weights[weightsCount] = {1, 2, 3};
- for (size_t i = 0; i < weightsCount; ++i) {
- wi.SetWeight(i, weights[i]);
- wi.Enable(i);
- }
- UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), weightsCount);
- UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 6);
- unsigned cnt[weightsCount] = {};
- for (size_t i = 0; i < wi.WeightsSum(); ++i) {
- const size_t choice = wi.Choose(i);
- UNIT_ASSERT(choice < weightsCount); // it is index
- ++cnt[choice];
- }
- for (size_t i = 0; i < weightsCount; ++i) {
- UNIT_ASSERT_VALUES_EQUAL_C(weights[i], cnt[i], "i: " << i);
- }
- }
-}
-
-Y_UNIT_TEST_SUITE(TMultiClusterProducerTest) {
- Y_UNIT_TEST(NotStartedProducerCanBeDestructed) {
- // Test that producer doesn't hang on till shutdown
- TPQLib lib;
- TMultiClusterProducerSettings settings;
- settings.SourceIdPrefix = "prefix";
- settings.ServerWeights.resize(2);
-
- {
- auto& w = settings.ServerWeights[0];
- w.Weight = 100500;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost"};
- s.Topic = "topic1";
- s.MaxAttempts = 10;
- }
-
- {
- auto& w = settings.ServerWeights[1];
- w.Weight = 1;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost"};
- s.Topic = "topic2";
- s.MaxAttempts = 10;
- }
- lib.CreateMultiClusterProducer(settings, {}, false);
- }
-
- static void InitServer(TTestServer& testServer) {
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2);
-
- testServer.WaitInit("topic1");
- }
-
- static void ConnectsToMinOnlineDcs(size_t min) {
- return; // Test is ignored. FIX: KIKIMR-7886
-
- TTestServer testServer(false);
- InitServer(testServer);
-
- TMultiClusterProducerSettings settings;
- settings.SourceIdPrefix = "prefix";
- settings.ServerWeights.resize(2);
- settings.MinimumWorkingDcsCount = min;
-
- {
- auto& w = settings.ServerWeights[0];
- w.Weight = 100500;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost", testServer.PortManager->GetPort()}; // port with nothing up
- s.Topic = "topic1";
- s.ReconnectionDelay = TDuration::MilliSeconds(1);
- s.MaxAttempts = 10;
- }
-
- {
- auto& w = settings.ServerWeights[1];
- w.Weight = 1;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost", testServer.GrpcPort};
- s.Topic = "topic1";
- s.MaxAttempts = 10;
- }
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib lib;
- auto producer = lib.CreateMultiClusterProducer(settings, logger, false);
- auto start = producer->Start();
-
- if (min == 1) {
- UNIT_ASSERT_C(!start.GetValueSync().Response.HasError(), start.GetValueSync().Response);
- } else {
- UNIT_ASSERT_C(start.GetValueSync().Response.HasError(), start.GetValueSync().Response);
- }
-
- DestroyAndWait(producer);
- }
-
- Y_UNIT_TEST(ConnectsToMinOnlineDcs) {
- ConnectsToMinOnlineDcs(1);
- }
-
- Y_UNIT_TEST(FailsToConnectToMinOnlineDcs) {
- return; // Test is ignored. FIX: KIKIMR-7886
-
- ConnectsToMinOnlineDcs(2);
- }
-
- TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) {
- TConsumerSettings consumerSettings;
- consumerSettings.ClientId = "user";
- consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- consumerSettings.Topics.push_back("topic1");
- consumerSettings.CommitsDisabled = true;
- return consumerSettings;
- }
-
- void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response);
- }
-
- void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response);
- }
-
- std::pair<size_t, size_t> ReadNMessagesAndDestroyConsumers(THolder<IConsumer> consumer1, THolder<IConsumer> consumer2, size_t n) {
- std::atomic<size_t> readsDone(0), reads1(0), reads2(0);
- TSystemEvent ev;
- std::pair<std::atomic<size_t>*, IConsumer*> infos[] = {
- std::make_pair(&reads1, consumer1.Get()),
- std::make_pair(&reads2, consumer2.Get())
- };
- for (size_t i = 0; i < n; ++i) {
- for (auto& info : infos) {
- auto* reads = info.first;
- info.second->GetNextMessage().Subscribe([&, reads](const NThreading::TFuture<TConsumerMessage>& respFuture) {
- ++(*reads);
- const size_t newReadsDone = ++readsDone;
- if (newReadsDone == n) {
- ev.Signal();
- }
- if (newReadsDone <= n) {
- AssertReadValid(respFuture);
- }
- });
- }
- }
- ev.Wait();
- consumer1 = nullptr;
- consumer2 = nullptr;
- return std::pair<size_t, size_t>(reads1, reads2);
- }
-
- Y_UNIT_TEST(WritesInSubproducers) {
- return; // Test is ignored. FIX: KIKIMR-7886
-
- TTestServer testServer1(false);
- InitServer(testServer1);
-
- TTestServer testServer2(false);
- InitServer(testServer2);
-
- TMultiClusterProducerSettings settings;
- settings.SourceIdPrefix = "producer";
- settings.ServerWeights.resize(2);
- settings.MinimumWorkingDcsCount = 1;
-
- {
- auto& w = settings.ServerWeights[0];
- w.Weight = 1;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost", testServer1.GrpcPort};
- s.Topic = "topic1";
- s.SourceId = "src0";
- s.MaxAttempts = 5;
- s.ReconnectionDelay = TDuration::MilliSeconds(30);
- }
-
- {
- auto& w = settings.ServerWeights[1];
- w.Weight = 1;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost", testServer2.GrpcPort};
- s.Topic = "topic1";
- s.SourceId = "src1";
- s.MaxAttempts = 2;
- s.ReconnectionDelay = TDuration::MilliSeconds(30);
- }
-
- TPQLibSettings pqLibSettings;
- pqLibSettings.ThreadsCount = 1;
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib lib(pqLibSettings);
- auto producer = lib.CreateMultiClusterProducer(settings, logger, false);
- auto startProducerResult = producer->Start();
- UNIT_ASSERT_C(!startProducerResult.GetValueSync().Response.HasError(), startProducerResult.GetValueSync().Response);
-
- auto consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false);
- UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError());
-
- auto consumer2 = lib.CreateConsumer(MakeConsumerSettings(testServer2), logger, false);
- UNIT_ASSERT(!consumer2->Start().GetValueSync().Response.HasError());
-
- const size_t writes = 100;
- for (size_t i = 0; i < writes; ++i) {
- auto write = producer->Write(TString(TStringBuilder() << "blob_1_" << i));
- AssertWriteValid(write);
- }
-
- const std::pair<size_t, size_t> msgs = ReadNMessagesAndDestroyConsumers(std::move(consumer1), std::move(consumer2), writes);
- UNIT_ASSERT_C(msgs.first > 30, "msgs.first = " << msgs.first);
- UNIT_ASSERT_C(msgs.second > 30, "msgs.second = " << msgs.second);
-
- consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false);
- UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError());
-
- testServer2.ShutdownServer();
- //testServer2.CleverServer->ShutdownGRpc();
-
- // Assert that all writes go in 1st DC
- for (size_t i = 0; i < writes; ++i) {
- auto write = producer->Write(TString(TStringBuilder() << "blob_2_" << i));
- auto read = consumer1->GetNextMessage();
- AssertWriteValid(write);
- AssertReadValid(read);
- }
-
- // Assert that all futures will be signalled after producer's death
- std::vector<NThreading::TFuture<TProducerCommitResponse>> writesFutures(writes);
- for (size_t i = 0; i < writes; ++i) {
- writesFutures[i] = producer->Write(TString(TStringBuilder() << "blob_3_" << i));
- }
-
- auto isDead = producer->IsDead();
- producer = nullptr;
- for (auto& f : writesFutures) {
- f.GetValueSync();
- }
- isDead.GetValueSync();
-
- DestroyAndWait(consumer1);
- DestroyAndWait(consumer2);
- }
-
- Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) {
- return; // Test is ignored. FIX: KIKIMR-7886
-
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer();
-
- const size_t partitions = 1;
- testServer.AnnoyingClient->InitRoot();
- testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
- testServer.AnnoyingClient->InitSourceIds();
-
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- TMultiClusterProducerSettings settings;
- settings.SourceIdPrefix = "producer";
- settings.ServerWeights.resize(2);
- settings.MinimumWorkingDcsCount = 1;
-
- for (auto& w : settings.ServerWeights) {
- w.Weight = 1;
- auto& s = w.ProducerSettings;
- s.ReconnectOnFailure = true;
- s.Server = TServerSetting{"localhost", testServer.GrpcPort};
- s.Topic = "topic1";
- s.SourceId = "src0";
- s.MaxAttempts = 5;
- s.ReconnectionDelay = TDuration::MilliSeconds(30);
- }
-
- auto producer = PQLib->CreateMultiClusterProducer(settings, logger, false);
- UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError());
- auto isDead = producer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto write1 = producer->Write(1, TString("blob1"));
- auto write2 = producer->Write(2, TString("blob2"));
-
- PQLib = nullptr;
-
- UNIT_ASSERT(write1.HasValue());
- UNIT_ASSERT(write2.HasValue());
-
- auto write3 = producer->Write(3, TString("blob3"));
- UNIT_ASSERT(write3.HasValue());
- UNIT_ASSERT(write3.GetValue().Response.HasError());
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp
deleted file mode 100644
index cc1f46d835..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp
+++ /dev/null
@@ -1,540 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include "internals.h"
-#include "persqueue_p.h"
-#include "compat_producer.h"
-#include "consumer.h"
-#include "multicluster_consumer.h"
-#include "producer.h"
-#include "retrying_consumer.h"
-#include "retrying_producer.h"
-#include "multicluster_producer.h"
-#include "processor.h"
-#include "compressing_producer.h"
-#include "decompressing_consumer.h"
-#include "local_caller.h"
-#include "ydb_sdk_consumer.h"
-
-#include <util/digest/numeric.h>
-#include <util/system/thread.h>
-
-namespace NPersQueue {
-
-
-TString TPQLib::GetUserAgent() const {
- return Impl->GetUserAgent();
-}
-
-void TPQLib::SetUserAgent(const TString& userAgent) {
- Impl->SetUserAgent(userAgent);
-}
-
-TPQLib::TPQLib(const TPQLibSettings& settings)
- : Impl(new TPQLibPrivate(settings))
-{}
-
-TPQLib::~TPQLib() {
- AtomicSet(Alive, 0);
- Impl->CancelObjectsAndWait();
-}
-
-#define CHECK_PQLIB_ALIVE() Y_VERIFY(AtomicGet(Alive), "Attempt to use PQLib after/during its destruction.")
-
-THolder<IProducer> TPQLib::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) {
- CHECK_PQLIB_ALIVE();
- return MakeHolder<TPublicProducer>(Impl->CreateMultiClusterProducer(settings, deprecated, std::move(logger)));
-}
-
-THolder<IProducer> TPQLib::CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) {
- CHECK_PQLIB_ALIVE();
- auto producer = settings.ReconnectOnFailure ?
- Impl->CreateRetryingProducer(settings, deprecated, std::move(logger)) :
- Impl->CreateProducer(settings, deprecated, std::move(logger));
- return MakeHolder<TPublicProducer>(std::move(producer));
-}
-
-template <class TConsumerOrProducerSettings>
-NYdb::NPersQueue::TPersQueueClient& TPQLibPrivate::GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger) {
- with_lock(Lock) {
- if (!CompressExecutor) {
- CompressExecutor = NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(CompressionPool);
- HandlersExecutor = NYdb::NPersQueue::CreateThreadPoolExecutor(1);
- }
- TWrapperKey key{settings.Server, settings.CredentialsProvider};
- auto it = Wrappers.find(key);
- if (Wrappers.end() == it) {
- CreateWrapper(key, logger);
- it = Wrappers.find(key);
- }
- return *it->second.PersQueueClient;
- }
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger)
-{
- if (Settings.EnableGRpcV1 && settings.Server.UseLogbrokerCDS != EClusterDiscoveryUsageMode::DontUse && !settings.CommitsDisabled && settings.Unpack) {
- auto ret = std::make_shared<TLocalConsumerImplCaller<TYdbSdkCompatibilityConsumer>>(settings,
- GetSelfRefsAreDeadEvent(false),
- this,
- logger,
- GetOrCreatePersQueueClient(settings, logger));
- ret->Init();
- AddToDestroySet(ret);
- return ret;
- }
- return nullptr;
-}
-
-THolder<IConsumer> TPQLib::CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) {
- CHECK_PQLIB_ALIVE();
- return MakeHolder<TPublicConsumer>(Impl->CreateConsumer(settings, deprecated, std::move(logger)));
-}
-
-THolder<IProcessor> TPQLib::CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) {
- CHECK_PQLIB_ALIVE();
- return MakeHolder<TPublicProcessor>(Impl->CreateProcessor(settings, deprecated, std::move(logger)));
-}
-
-void TPQLib::SetLogger(TIntrusivePtr<ILogger> logger) {
- CHECK_PQLIB_ALIVE();
- Impl->SetLogger(std::move(logger));
-}
-
-#undef CHECK_PQLIB_ALIVE
-
-static void DoExecute(std::shared_ptr<grpc::CompletionQueue> cq, std::shared_ptr<std::atomic<bool>> shuttingDown) {
- TThread::SetCurrentThreadName("pqlib_grpc_thr");
- void* tag;
- bool ok;
- while (cq->Next(&tag, &ok)) {
- IQueueEvent* const ev(static_cast<IQueueEvent*>(tag));
- if (shuttingDown->load() || !ev->Execute(ok)) {
- ev->DestroyRequest();
- }
- }
-}
-
-TPQLibPrivate::TPQLibPrivate(const TPQLibSettings& settings)
- : Settings(settings)
- , CQ(std::make_shared<grpc::CompletionQueue>())
- , Scheduler(MakeHolder<TScheduler>(this))
- , Logger(settings.DefaultLogger)
-{
- Y_VERIFY(Settings.ThreadsCount > 0);
- Y_VERIFY(Settings.GRpcThreads > 0);
- Y_VERIFY(Settings.CompressionPoolThreads > 0);
- CompressionPool = std::make_shared<TThreadPool>();
- CompressionPool->Start(Settings.CompressionPoolThreads);
- QueuePool.Start(Settings.ThreadsCount);
- GRpcThreads.reserve(Settings.GRpcThreads);
- for (size_t i = 0; i < Settings.GRpcThreads; ++i) {
- GRpcThreads.emplace_back([cq = CQ, shuttingDown = ShuttingDown](){ DoExecute(cq, shuttingDown); });
- }
- CreateSelfRefsAreDeadPtr();
-}
-
-TPQLibPrivate::~TPQLibPrivate() {
- DEBUG_LOG("Destroying PQLib. Destroying scheduler.", "", "");
- Scheduler.Reset();
-
- DEBUG_LOG("Destroying PQLib. Set stop flag.", "", "");
- Stop();
-
- if (CQ) {
- DEBUG_LOG("Destroying PQLib. Shutting down completion queue.", "", "");
- CQ->Shutdown();
- }
-
- if (!AtomicGet(HasDeprecatedObjects)) {
- DEBUG_LOG("Destroying PQLib. Joining threads.", "", "");
- for (std::thread& thread : GRpcThreads) {
- thread.join();
- }
- } else {
- DEBUG_LOG("Destroying PQLib. Detaching threads.", "", "");
- for (std::thread& thread : GRpcThreads) {
- thread.detach();
- }
- }
-
- if (YdbDriver) {
- YdbDriver->Stop();
- YdbDriver = nullptr;
- }
-
- DEBUG_LOG("Destroying PQLib. Stopping compression pool.", "", "");
- CompressionPool->Stop();
-
- DEBUG_LOG("Destroying PQLib. Stopping queue pool.", "", "");
- QueuePool.Stop();
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- std::shared_ptr<IProducerImpl> res;
- auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger));
- rawProducerImpl->SetChannel(channelInfo);
- res = std::move(rawProducerImpl);
- return res;
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- TChannelHolder t = CreateChannel(settings, ChooseLogger(logger));
- std::shared_ptr<IProducerImpl> res;
-
- auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger));
- rawProducerImpl->SetChannel(t);
- res = std::move(rawProducerImpl);
- return res;
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create producer", settings.SourceId, "");
- std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger));
- compressing->Init();
- if (!deprecated) {
- AddToDestroySet(compressing);
- }
- return compressing;
-}
-
-class TLoggerWrapper : public TLogBackend {
-public:
- TLoggerWrapper(TIntrusivePtr<ILogger> logger) noexcept
- : Logger(logger)
- {}
-
- ~TLoggerWrapper() {}
-
- void WriteData(const TLogRecord& rec)
- {
- Logger->Log(TString(rec.Data, rec.Len), "", "", (int)rec.Priority);
- }
-
- void ReopenLog() {}
-
-private:
- TIntrusivePtr<ILogger> Logger;
-};
-
-
-void TPQLibPrivate::CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger) {
- auto& wrapper = Wrappers[key];
- const auto& settings = key.Settings;
- if (!YdbDriver) {
- NYdb::TDriverConfig config;
-
- config.SetEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port) //TODO BAD
- .SetDatabase(settings.Database.empty() ? "/Root" : settings.Database)
- .SetNetworkThreadsNum(Settings.GRpcThreads)
- .SetClientThreadsNum(Settings.ThreadsCount)
- .SetGRpcKeepAliveTimeout(TDuration::Seconds(90))
- .SetGRpcKeepAlivePermitWithoutCalls(true)
- .SetDiscoveryMode(NYdb::EDiscoveryMode::Async);
- if (logger)
- config.SetLog(MakeHolder<TLoggerWrapper>(logger));
- if (settings.UseSecureConnection)
- config.UseSecureConnection(settings.CaCert);
-
- YdbDriver = MakeHolder<NYdb::TDriver>(config);
- }
-
- NYdb::NPersQueue::TPersQueueClientSettings pqSettings;
- pqSettings.DefaultCompressionExecutor(CompressExecutor)
- .DefaultHandlersExecutor(HandlersExecutor)
- .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto)
- .DiscoveryEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port)
- .Database(settings.Database);
- if (key.Provider) {
- pqSettings.CredentialsProviderFactory(std::shared_ptr<NYdb::ICredentialsProviderFactory>(new TCredentialsProviderFactoryWrapper(key.Provider)));
- }
- wrapper.PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*YdbDriver, pqSettings);
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger)
-{
- if (Settings.EnableGRpcV1) {
- auto ret = std::make_shared<TLocalProducerImplCaller<TYdbSdkCompatibilityProducer>>(settings,
- GetOrCreatePersQueueClient(settings, logger),
- GetSelfRefsAreDeadEvent(false),
- this);
- ret->Init();
- AddToDestroySet(ret);
- return ret;
- }
- return nullptr;
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- auto newP = CreateNewProducer(settings, ChooseLogger(logger));
- if (newP) return newP;
-
- DEBUG_LOG("Create producer", settings.SourceId, "");
- std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger));
- compressing->Init();
- if (!deprecated) {
- AddToDestroySet(compressing);
- }
- return compressing;
-}
-
-template<typename TRawProducerImpl>
-std::shared_ptr<TRawProducerImpl> TPQLibPrivate::CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create raw producer", settings.SourceId, "");
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- NThreading::TPromise<TProducerCreateResponse> promise = NThreading::NewPromise<TProducerCreateResponse>();
- auto ret = std::make_shared<TLocalProducerImplCaller<TRawProducerImpl>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger));
- ret->Init();
- if (addToDestroySet) {
- AddToDestroySet(ret);
- }
- return ret;
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- auto t = CreateChannel(settings.Server, settings.CredentialsProvider, ChooseLogger(logger), settings.PreferLocalProxy);
- auto res = CreateRawConsumerImpl(settings, std::move(refsAreDeadPtr), ChooseLogger(logger));
- res->SetChannel(t);
- return std::move(res);
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- auto res = CreateRawConsumerImpl(settings, refsAreDeadPtr, ChooseLogger(logger));
- res->SetChannel(channelInfo);
- return std::move(res);
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create multicluster consumer", "", "");
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- auto consumer = std::make_shared<TLocalConsumerImplCaller<TMultiClusterConsumer>>(settings, std::move(refsAreDeadPtr), this, logger);
- consumer->Init();
- if (addToDestroySet) {
- AddToDestroySet(consumer);
- }
- return std::move(consumer);
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) {
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(std::move(subconsumer), settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger));
- consumer->Init();
- if (addToDestroySet) {
- AddToDestroySet(consumer);
- }
- return consumer;
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) {
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TRetryingConsumer>>(settings, std::move(refsAreDeadPtr), this, logger);
- consumer->Init();
- if (addToDestroySet) {
- AddToDestroySet(consumer);
- }
- return consumer;
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- if (auto newConsumer = CreateNewConsumer(settings, ChooseLogger(logger))) {
- return newConsumer;
- }
-
- DEBUG_LOG("Create consumer", "", "");
- // Create raw consumer.
- std::shared_ptr<IConsumerImpl> consumer;
- if (settings.ReadFromAllClusterSources) {
- if (settings.ReadMirroredPartitions) {
- ythrow yexception() << "Can't create consumer with both ReadMirroredPartitions and ReadFromAllClusterSources options";
- }
- if (settings.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) {
- ythrow yexception() << "Can't create consumer with ReadFromAllClusterSources option, but without using cluster discovery";
- }
- consumer = CreateRawMultiClusterConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- } else if (settings.ReconnectOnFailure) {
- consumer = CreateRawRetryingConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- } else {
- consumer = CreateRawConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- }
- if (settings.Unpack) {
- consumer = CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- }
- return consumer;
-}
-
-std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create consumer", "", "");
- std::shared_ptr<IConsumerImpl> consumer = CreateRawConsumer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- if (settings.Unpack) {
- return CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- }
- return consumer;
-}
-
-std::shared_ptr<TConsumer> TPQLibPrivate::CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create raw consumer", "", "");
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- NThreading::TPromise<TConsumerCreateResponse> promise = NThreading::NewPromise<TConsumerCreateResponse>();
- std::shared_ptr<TConsumer> consumer = std::make_shared<TLocalConsumerImplCaller<TConsumer>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger));
- consumer->Init();
- if (addToDestroySet) {
- AddToDestroySet(consumer);
- }
- return consumer;
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create raw retrying producer", settings.SourceId, "");
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- auto producer = std::make_shared<TLocalProducerImplCaller<TRetryingProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger));
- producer->Init();
- if (addToDestroySet) {
- AddToDestroySet(producer);
- }
- return std::move(producer);
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- auto newP = CreateNewProducer(settings, ChooseLogger(logger));
- if (newP) return newP;
-
- DEBUG_LOG("Create retrying producer", settings.SourceId, "");
- std::shared_ptr<IProducerImpl> rawProducer = CreateRawRetryingProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger));
- compressing->Init();
- if (!deprecated) {
- AddToDestroySet(compressing);
- }
- return compressing;
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create raw multicluster producer", settings.SourceIdPrefix, "");
- const bool addToDestroySet = refsAreDeadPtr != nullptr;
- auto producer = std::make_shared<TLocalProducerImplCaller<TMultiClusterProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger));
- producer->Init();
- if (addToDestroySet) {
- AddToDestroySet(producer);
- }
- return std::move(producer);
-}
-
-std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create multicluster producer", settings.SourceIdPrefix, "");
- if (settings.ServerWeights.empty()) {
- ythrow yexception() << "Can't create producer without server";
- }
- const ECodec codec = settings.ServerWeights[0].ProducerSettings.Codec;
- const int quality = settings.ServerWeights[0].ProducerSettings.Quality;
- for (const auto& w : settings.ServerWeights) {
- if (w.ProducerSettings.Codec != codec) {
- ythrow yexception() << "Can't create multicluster producer with different codecs";
- }
- }
-
- std::shared_ptr<IProducerImpl> rawProducer = CreateRawMultiClusterProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger));
- std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), codec, quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger));
- compressing->Init();
- if (!deprecated) {
- AddToDestroySet(compressing);
- }
- return compressing;
-}
-
-std::shared_ptr<IProcessorImpl> TPQLibPrivate::CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger)
-{
- DEBUG_LOG("Create processor", "", "");
- auto processor = std::make_shared<TLocalProcessorImplCaller<TProcessor>>(settings, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger));
- processor->Init();
- if (!deprecated) {
- AddToDestroySet(processor);
- }
- return processor;
-}
-
-TChannelHolder TPQLibPrivate::CreateChannel(
- const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TIntrusivePtr<ILogger> logger,
- bool preferLocalProxy
-) {
- DEBUG_LOG("Create channel", "", "");
- TChannelHolder res;
- res.ChannelPtr = new TChannel(server, credentialsProvider, this, ChooseLogger(logger), preferLocalProxy);
- res.ChannelInfo = res.ChannelPtr->GetChannel();
- res.ChannelPtr->Start();
- return res;
-}
-
-
-TChannelHolder TPQLibPrivate::CreateChannel(
- const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool preferLocalProxy
-) {
- DEBUG_LOG("Create channel", "", "");
- TChannelHolder res;
- res.ChannelPtr = new TChannel(settings, this, ChooseLogger(logger), preferLocalProxy);
- res.ChannelInfo = res.ChannelPtr->GetChannel();
- res.ChannelPtr->Start();
- return res;
-}
-
-void TPQLibPrivate::CancelObjectsAndWait() {
- {
- auto guard = Guard(Lock);
- DEBUG_LOG("Destroying PQLib. Cancelling objects: " << SyncDestroyedSet.size(), "", "");
- for (auto& ptr : SyncDestroyedSet) {
- auto ref = ptr.lock();
- if (ref) {
- ref->Cancel();
- }
- }
- SelfRefsAreDeadPtr = nullptr;
- }
- DEBUG_LOG("Destroying PQLib. Waiting refs to die", "", "");
- SelfRefsAreDeadEvent.Wait();
-
- DEBUG_LOG("Destroying PQLib. HasDeprecatedObjects: " << AtomicGet(HasDeprecatedObjects) << ". Refs to PQLibPrivate: " << RefCount(), "", "");
- if (!AtomicGet(HasDeprecatedObjects)) {
- Y_ASSERT(RefCount() == 1);
- }
-}
-
-void TPQLibPrivate::CreateSelfRefsAreDeadPtr() {
- void* fakeAddress = &SelfRefsAreDeadEvent;
- SelfRefsAreDeadPtr = std::shared_ptr<void>(fakeAddress,
- [event = SelfRefsAreDeadEvent](void*) mutable {
- event.Signal();
- });
-}
-
-void TPQLibPrivate::AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj) {
- auto guard = Guard(Lock);
- // Check dead objects first
- if (DestroySetAddCounter * 2 > SyncDestroyedSet.size()) {
- for (auto i = SyncDestroyedSet.begin(); i != SyncDestroyedSet.end();) {
- if (i->expired()) {
- i = SyncDestroyedSet.erase(i);
- } else {
- ++i;
- }
- }
- DestroySetAddCounter = 0;
- }
- ++DestroySetAddCounter;
- SyncDestroyedSet.emplace(std::move(obj));
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h
deleted file mode 100644
index 4c7145b048..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h
+++ /dev/null
@@ -1,316 +0,0 @@
-#pragma once
-
-#include "channel.h"
-#include "internals.h"
-#include "iconsumer_p.h"
-#include "iprocessor_p.h"
-#include "iproducer_p.h"
-#include "scheduler.h"
-#include "queue_pool.h"
-
-#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-#include <util/generic/maybe.h>
-#include <util/generic/queue.h>
-#include <util/generic/guid.h>
-#include <util/generic/hash_set.h>
-#include <util/system/types.h>
-#include <util/system/mutex.h>
-#include <util/thread/factory.h>
-#include <util/thread/pool.h>
-
-#include <util/datetime/base.h>
-
-#include <grpc++/create_channel.h>
-#include <grpc++/completion_queue.h>
-#include <google/protobuf/message.h>
-
-#include <atomic>
-#include <type_traits>
-#include <memory>
-#include <set>
-#include <thread>
-
-namespace NPersQueue {
-
-class TProducer;
-class TConsumer;
-
-
-struct TWrapperKey {
- TServerSetting Settings;
- std::shared_ptr<ICredentialsProvider> Provider;
-
- bool operator <(const TWrapperKey& b) const {
- return Settings < b.Settings || (!(b.Settings < Settings) && Provider < b.Provider);
- }
-
-};
-
-
-
-class TCredentialsProviderWrapper : public NYdb::ICredentialsProvider {
-public:
- TCredentialsProviderWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider)
- : Provider(provider)
- {}
-
- bool IsValid() const override {
- return true;
- }
-
- TString GetAuthInfo() const override {
- return GetToken(Provider.get());
- }
-
- private:
- std::shared_ptr<NPersQueue::ICredentialsProvider> Provider;
-};
-
-
-class TCredentialsProviderFactoryWrapper : public NYdb::ICredentialsProviderFactory {
-public:
- TString GetClientIdentity() const override {
- return Guid;
- }
-
- std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override {
- return Provider;
- };
-
-
- TCredentialsProviderFactoryWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider)
- : Provider(new TCredentialsProviderWrapper(provider))
- , Guid(CreateGuidAsString())
- {}
-
-private:
- std::shared_ptr<NYdb::ICredentialsProvider> Provider;
- TString Guid;
-};
-
-class TPQLibPrivate: public TAtomicRefCount<TPQLibPrivate> {
- template <class T>
- class TBaseFutureSubscriberObjectInQueue: public IObjectInQueue {
- public:
- void SetFuture(const NThreading::TFuture<T>& future) {
- Future = future;
- }
-
- protected:
- NThreading::TFuture<T> Future;
- };
-
- template <class T, class TFunc>
- class TFutureSubscriberObjectInQueue: public TBaseFutureSubscriberObjectInQueue<T> {
- public:
- template <class TF>
- TFutureSubscriberObjectInQueue(TF&& f)
- : Func(std::forward<TF>(f))
- {
- }
-
- void Process(void*) override {
- THolder<IObjectInQueue> holder(this); // like in TOwnedObjectInQueue::Process()
- Func(this->Future);
- }
-
- private:
- TFunc Func;
- };
-
- template <class T, class TFunc>
- static THolder<TBaseFutureSubscriberObjectInQueue<T>> MakeFutureSubscribeObjectInQueue(TFunc&& f) {
- return MakeHolder<TFutureSubscriberObjectInQueue<T, std::remove_reference_t<TFunc>>>(std::forward<TFunc>(f));
- }
-
- template <class T>
- class TSubscriberFunc {
- public:
- TSubscriberFunc(TPQLibPrivate* pqLib, const void* queueTag, THolder<TBaseFutureSubscriberObjectInQueue<T>> callback)
- : PQLib(pqLib)
- , QueueTag(queueTag)
- , Callback(std::make_shared<THolder<TBaseFutureSubscriberObjectInQueue<T>>>(std::move(callback)))
- {
- }
-
- void operator()(const NThreading::TFuture<T>& future) {
- if (Callback) {
- (*Callback)->SetFuture(future);
- THolder<IObjectInQueue> callback(std::move(*Callback));
- const bool added = PQLib->GetQueuePool().GetQueue(QueueTag).Add(callback.Get());
- if (added) { // can be false on shutdown
- Y_UNUSED(callback.Release());
- }
- }
- }
-
- private:
- TPQLibPrivate* PQLib;
- const void* QueueTag;
- // std::shared_ptr is only because copy constructor is required
- std::shared_ptr<THolder<TBaseFutureSubscriberObjectInQueue<T>>> Callback;
- };
-
-public:
- TPQLibPrivate(const TPQLibSettings& settings);
- TPQLibPrivate(const TPQLibPrivate&) = delete;
- TPQLibPrivate(TPQLibPrivate&&) = delete;
- ~TPQLibPrivate();
-
- TChannelHolder CreateChannel(const TServerSetting& server, const TCredProviderPtr& credentialsProvider,
- TIntrusivePtr<ILogger> logger, bool preferLocalProxy = false);
- TChannelHolder CreateChannel(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger,
- bool preferLocalProxy = false);
-
-
- std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IProducerImpl> CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IProducerImpl> CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IProducerImpl> CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IProducerImpl> CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IProducerImpl> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IConsumerImpl> CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IConsumerImpl> CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IConsumerImpl> CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger);
-
- std::shared_ptr<IProcessorImpl> CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger);
-
- TString GetUserAgent() const {
- auto guard = Guard(UALock);
- return UserAgent;
- }
-
- void SetUserAgent(const TString& userAgent) {
- auto guard = Guard(UALock);
- UserAgent = userAgent;
- }
-
- void Stop() {
- ShuttingDown->store(true);
- }
-
- const std::shared_ptr<grpc::CompletionQueue>& GetCompletionQueue() const {
- return CQ;
- }
-
- const TPQLibSettings& GetSettings() const {
- return Settings;
- }
-
- TScheduler& GetScheduler() {
- return *Scheduler;
- }
-
- TQueuePool& GetQueuePool() {
- return QueuePool;
- }
-
- IThreadPool& GetCompressionPool() {
- return *CompressionPool;
- }
-
- template <class TFunc, class T>
- void Subscribe(const NThreading::TFuture<T>& future, const void* queueTag, TFunc&& func) {
- TSubscriberFunc<T> f(this, queueTag, MakeFutureSubscribeObjectInQueue<T>(std::forward<TFunc>(func)));
- future.Subscribe(std::move(f));
- }
-
- void SetLogger(TIntrusivePtr<ILogger> logger) {
- Logger = std::move(logger);
- }
-
- TIntrusivePtr<ILogger> ChooseLogger(TIntrusivePtr<ILogger> logger) {
- return logger ? std::move(logger) : Logger;
- }
-
- void CancelObjectsAndWait();
-
- // for testing
- std::shared_ptr<void> GetSelfRefsAreDeadPtr() const {
- return SelfRefsAreDeadPtr;
- }
-
- void AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj);
-
-private:
- template<typename TRawProducerImpl>
- std::shared_ptr<TRawProducerImpl> CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<TConsumer> CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
- std::shared_ptr<IConsumerImpl> CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger);
-
- void CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger);
-
- template <class TConsumerOrProducerSettings>
- NYdb::NPersQueue::TPersQueueClient& GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger);
-
- std::shared_ptr<void> GetSelfRefsAreDeadEvent(bool isObjectDeprecated) {
- if (isObjectDeprecated) {
- AtomicSet(HasDeprecatedObjects, 1);
- }
- return isObjectDeprecated ? nullptr : SelfRefsAreDeadPtr;
- }
-
- void CreateSelfRefsAreDeadPtr();
-
- using IThreadRef = TAutoPtr<IThreadFactory::IThread>;
-
-private:
- const TPQLibSettings Settings;
-
- std::shared_ptr<grpc::CompletionQueue> CQ;
- std::vector<std::thread> GRpcThreads;
-
- std::shared_ptr<std::atomic<bool>> ShuttingDown = std::make_shared<std::atomic<bool>>(false); // shared_ptr is for deprecated case when we have leaked threads
-
- THolder<TScheduler> Scheduler;
- TQueuePool QueuePool;
- std::shared_ptr<IThreadPool> CompressionPool;
-
- TIntrusivePtr<ILogger> Logger;
-
- TAtomic HasDeprecatedObjects = 0;
-
- TAutoEvent SelfRefsAreDeadEvent;
- std::shared_ptr<void> SelfRefsAreDeadPtr;
- TAdaptiveLock Lock, UALock;
- size_t DestroySetAddCounter = 0;
- std::set<std::weak_ptr<TSyncDestroyed>, std::owner_less<std::weak_ptr<TSyncDestroyed>>> SyncDestroyedSet;
-
- TString UserAgent = "C++ pqlib v0.3";
-
- struct TYdbWrapper {
- THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient;
- };
-
- THolder<NYdb::TDriver> YdbDriver;
- std::map<TWrapperKey, TYdbWrapper> Wrappers;
- NYdb::NPersQueue::IExecutor::TPtr CompressExecutor;
- NYdb::NPersQueue::IExecutor::TPtr HandlersExecutor;
- std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactoryWrapper;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp
deleted file mode 100644
index a695c2a8da..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "persqueue_p.h"
-#include "producer.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NPersQueue {
-}
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp
deleted file mode 100644
index 1b3ca57fd8..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp
+++ /dev/null
@@ -1,358 +0,0 @@
-#include "processor.h"
-#include <util/string/builder.h>
-
-namespace NPersQueue {
-
-template<typename T>
-bool IsReady(const NThreading::TFuture<T>& future)
-{
- return future.HasValue() || future.HasException();
-}
-
-template<typename T>
-void Reset(NThreading::TFuture<T>& future)
-{
- future = NThreading::TFuture<T>();
-}
-
-template<typename T>
-void TProcessor::SafeSubscribe(NThreading::TFuture<T>& future)
-{
- std::weak_ptr<TProcessor> self(shared_from_this());
- PQLib->Subscribe(
- future,
- this,
- [self](const auto&) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->ProcessFutures();
- }});
-}
-
-TProcessor::TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept
- : IProcessorImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , Logger(std::move(logger))
-{}
-
-TProcessor::~TProcessor() noexcept
-{
- SubscribeDestroyed();
- CleanState();
-}
-
-void TProcessor::Init()
-{
- auto selfShared = shared_from_this();
- auto init = [selfShared] {
- selfShared->InitImpl();
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(this).AddFunc(init));
-}
-
-void TProcessor::InitImpl()
-{
- ScheduleIdleSessionsCleanup();
- RecreateConsumer();
- SafeSubscribe(ConsumerIsStarted);
- SafeSubscribe(ConsumerIsDead);
-}
-
-// Must be called under lock
-void TProcessor::CleanState() noexcept
-{
- ProcessingData.clear();
- Reset(ConsumerIsDead);
- Reset(ConsumerRequest);
- Consumer.reset();
- Producers.clear();
- Data.clear();
- CurrentOriginDataMemoryUsage = 0;
- CurrentProcessedDataMemoryUsage = 0;
-}
-
-
-bool TProcessor::NextRequest() noexcept
-{
- Y_VERIFY(Consumer);
- Y_VERIFY(!ConsumerRequest.Initialized());
-
- DEBUG_LOG("Current memory usage: "
- << "origin messages " << CurrentOriginDataMemoryUsage << "(" << Settings.MaxOriginDataMemoryUsage << "), "
- << "processed messages " << CurrentProcessedDataMemoryUsage << "(" << Settings.MaxProcessedDataMemoryUsage << ")\n",
- "", "");
-
- if (CurrentOriginDataMemoryUsage < Settings.MaxOriginDataMemoryUsage && CurrentProcessedDataMemoryUsage < Settings.MaxProcessedDataMemoryUsage) {
- DEBUG_LOG("Issuing consumer request\n", "", "");
- ConsumerRequest = Consumer->GetNextMessage();
- return true;
- } else {
- return false;
- }
-}
-
-bool TProcessor::ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept
-{
- Y_VERIFY(ConsumerRequest.HasValue());
- auto message = ConsumerRequest.ExtractValueSync();
- Reset(ConsumerRequest);
- const auto& type = message.Type;
- const auto& resp = message.Response;
- if (type == EMT_ERROR) {
- // Will be handled later via ConsumerIsDead
- ERR_LOG("Got error: " << resp << "\n", "", "");
- } else if (type == EMT_RELEASE) {
- Y_VERIFY(resp.HasRelease());
- auto t = resp.GetRelease().GetTopic();
- auto p = resp.GetRelease().GetPartition();
- auto g = resp.GetRelease().GetGeneration();
- DEBUG_LOG("Got release for " << t << ":" << p << ", generation " << g << "\n", "", "");
- } else if (type == EMT_LOCK) {
- Y_VERIFY(resp.HasLock());
- auto t = resp.GetLock().GetTopic();
- auto p = resp.GetLock().GetPartition();
- auto g = resp.GetLock().GetGeneration();
- DEBUG_LOG("Got lock for " << t << ":" << p << ", generation " << g << "\n", "", "");
- message.ReadyToRead.SetValue(TLockInfo{});
- } else if (type == EMT_DATA) {
- Y_VERIFY(resp.HasData());
- Data.push_back(TDataInfo{resp.GetData().GetCookie()});
- for (auto& batch: resp.GetData().GetMessageBatch()) {
- TPartition partition(batch.GetTopic(), batch.GetPartition());
- TVector<TOriginMessage>& originMessages = result.Messages[partition];
- TDeque<TProcessingData>& processingData = ProcessingData[partition];
- Data.back().TotalOriginMessages += batch.MessageSize();
-
- for (auto& msg: batch.GetMessage()) {
- CurrentOriginDataMemoryUsage += msg.ByteSizeLong();
- NThreading::TPromise<TProcessedData> promise = NThreading::NewPromise<TProcessedData>();
- NThreading::TFuture<TProcessedData> future = promise.GetFuture();
- TOriginMessage originMessage = TOriginMessage{msg, promise};
- processedDataFutures.push_back(future);
- originMessages.push_back(originMessage);
- processingData.push_back(TProcessingData{future, msg.GetOffset(), msg.ByteSizeLong(), &Data.back()});
- }
- }
- DEBUG_LOG("Processing data message " << resp.GetData().GetCookie() << "\n", "", "");
- return true;
- } else if (type == EMT_COMMIT) {
- DEBUG_LOG("Got commit", "", "");
- } else {
- WARN_LOG("Unknown message type " << int(type) << ", response " << resp << "\n", "", "");
- }
-
- return false;
-}
-
-// Must be called under lock
-void TProcessor::RecreateConsumer() noexcept
-{
- if (DestroyedFlag) {
- return;
- }
-
- INFO_LOG("Create consumer\n", "", "");
- CleanState();
-
- TConsumerSettings consumerSettings = Settings.ConsumerSettings;
- consumerSettings.UseLockSession = true;
-
- Consumer = PQLib->CreateConsumer(consumerSettings, DestroyEventRef == nullptr, Logger);
- ConsumerIsStarted = Consumer->Start();
- ConsumerIsDead = Consumer->IsDead();
-}
-
-void TProcessor::ScheduleIdleSessionsCleanup() noexcept
-{
- if (DestroyedFlag) {
- return;
- }
-
- std::weak_ptr<TProcessor> self(shared_from_this());
- PQLib->GetScheduler().Schedule(Settings.SourceIdIdleTimeout,
- this,
- [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->CloseIdleSessions();
- }
- });
-}
-
-void TProcessor::CloseIdleSessions() noexcept
-{
- if (DestroyedFlag) {
- return;
- }
-
- TInstant now = TInstant::Now();
- for (auto it = Producers.begin(); it != Producers.end();) {
- if (it->second.LastWriteTime + Settings.SourceIdIdleTimeout < now) {
- INFO_LOG("Close producer for sourceid=" << it->first.SourceId, "", "");
- it = Producers.erase(it);
- } else {
- ++it;
- }
- }
- ScheduleIdleSessionsCleanup();
-}
-
-void TProcessor::ProcessFutures() noexcept
-{
- if (DestroyedFlag) {
- return;
- }
-
- TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture;
- TVector<NThreading::TFuture<TProcessedData>> processedDataFutures;
- TVector<NThreading::TFuture<TProducerCommitResponse>> producerAckFutures;
-
- if (IsReady(ConsumerIsDead)) {
- ERR_LOG("Consumer died with error" << ConsumerIsDead.ExtractValueSync(), "", "");
- RecreateConsumer();
- SafeSubscribe(ConsumerIsStarted);
- SafeSubscribe(ConsumerIsDead);
- return;
- }
-
- if (IsReady(ConsumerRequest)) {
- TOriginData data;
- bool hasData = ProcessConsumerResponse(data, processedDataFutures);
- if (hasData) {
- Requests.front().SetValue(data);
- Requests.pop_front();
- }
- }
-
- TInstant now = TInstant::Now();
- for (auto& partitionData: ProcessingData) {
- TDeque<TProcessingData>& processingData = partitionData.second;
-
- while (!processingData.empty() && IsReady(processingData.front().Processed)) {
- Y_VERIFY(processingData.front().Processed.HasValue(), "processing future cannot be filled with exception");
- ui64 originMessageOffset = processingData.front().OriginMessageOffset;
- CurrentOriginDataMemoryUsage -= processingData.front().OriginMessageSize;
- TProcessedData processedData = processingData.front().Processed.ExtractValueSync();
- TDataInfo* dataInfo = processingData.front().DataInfo;
- dataInfo->OriginMessagesCounter++;
- dataInfo->TotalProcessedMessages += processedData.Messages.size();
- processingData.pop_front();
- for (auto& processedMessage: processedData.Messages) {
- TString newSourceId = processedMessage.SourceIdPrefix + partitionData.first.Topic + ":" + ToString(partitionData.first.PartitionId) + "_" + processedMessage.Topic;
- TProducerKey producerKey { processedMessage.Topic, newSourceId };
- IProducer* producer = nullptr;
-
- auto it = Producers.find(producerKey);
- if (it == Producers.end()) {
- TProducerSettings producerSettings = Settings.ProducerSettings;
- producerSettings.Topic = processedMessage.Topic;
- producerSettings.SourceId = newSourceId;
- producerSettings.PartitionGroup = processedMessage.Group;
- producerSettings.ReconnectOnFailure = true;
- Producers[producerKey] = TProducerInfo { PQLib->CreateRetryingProducer(producerSettings, DestroyEventRef == nullptr, Logger), {}, now};
- producer = Producers[producerKey].Producer.get();
- producer->Start();
- } else {
- producer = it->second.Producer.get();
- }
-
- CurrentProcessedDataMemoryUsage += processedMessage.Data.size();
- DEBUG_LOG("Write message with seqNo=" << originMessageOffset + 1 << " to sourceId=" << newSourceId, "", "");
- NThreading::TFuture<TProducerCommitResponse> ack = producer->Write(originMessageOffset + 1, std::move(processedMessage.Data));
- producerAckFutures.push_back(ack);
- Producers[producerKey].Queue.push_back(TProduceInfo(ack, dataInfo));
- }
- }
- }
-
- for (auto& producer: Producers) {
- TProducerInfo& producerInfo = producer.second;
- while (!producerInfo.Queue.empty() && IsReady(producerInfo.Queue.front().Ack)) {
- auto ack = producerInfo.Queue.front().Ack.ExtractValueSync();
- DEBUG_LOG("Got ack for message with seqNo=" << ack.SeqNo << " from sourceId=" << producer.first.SourceId << " " << ack.Response, "", "");
- producerInfo.LastWriteTime = now;
- producerInfo.Queue.front().DataInfo->ProcessedMessagesCounter++;
- CurrentProcessedDataMemoryUsage -= ack.Data.GetSourceData().size();
- producerInfo.Queue.pop_front();
- }
- }
-
- TVector<ui64> toCommit;
- while (!Data.empty() && Data.front().IsReadyForCommit()) {
- toCommit.push_back(Data.front().Cookie);
- Data.pop_front();
- }
- if (!toCommit.empty()) {
- Consumer->Commit(toCommit);
- }
-
- if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized() && !Requests.empty()) {
- if (NextRequest()) {
- consumerRequestFuture = ConsumerRequest;
- }
- }
-
- if (consumerRequestFuture.Defined()) {
- SafeSubscribe(*consumerRequestFuture.Get());
- }
-
- for (auto& future: processedDataFutures) {
- SafeSubscribe(future);
- }
-
- for (auto& future: producerAckFutures) {
- SafeSubscribe(future);
- }
-}
-
-void TProcessor::GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept
-{
- Y_VERIFY(!DestroyedFlag);
- TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture;
- Requests.push_back(promise);
- if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized()) {
- if (NextRequest()) {
- consumerRequestFuture = ConsumerRequest;
- }
- }
-
- if (consumerRequestFuture.Defined()) {
- SafeSubscribe(*consumerRequestFuture.Get());
- }
-}
-
-void TProcessor::SubscribeDestroyed() {
- NThreading::TPromise<void> promise = ObjectsDestroyed;
- auto handler = [promise](const auto&) mutable {
- promise.SetValue();
- };
- std::vector<NThreading::TFuture<void>> futures;
- futures.reserve(Producers.size() + 2);
- futures.push_back(DestroyedPromise.GetFuture());
- if (Consumer) {
- futures.push_back(Consumer->Destroyed());
- }
- for (const auto& p : Producers) {
- if (p.second.Producer) {
- futures.push_back(p.second.Producer->Destroyed());
- }
- }
- WaitExceptionOrAll(futures).Subscribe(handler);
-}
-
-NThreading::TFuture<void> TProcessor::Destroyed() noexcept {
- return ObjectsDestroyed.GetFuture();
-}
-
-void TProcessor::Cancel() {
- DestroyedFlag = true;
-
- for (auto& req : Requests) {
- req.SetValue(TOriginData());
- }
- Requests.clear();
-
- DestroyPQLibRef();
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h
deleted file mode 100644
index a6abb827a0..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h
+++ /dev/null
@@ -1,118 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include "persqueue_p.h"
-
-#include <library/cpp/threading/future/future.h>
-#include <util/generic/deque.h>
-#include <util/generic/map.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-
-namespace NPersQueue {
-
-struct TDataInfo {
- ui64 Cookie;
- ui64 TotalOriginMessages;
- ui64 OriginMessagesCounter;
- ui64 TotalProcessedMessages;
- ui64 ProcessedMessagesCounter;
-
- TDataInfo(ui64 cookie)
- : Cookie(cookie)
- , TotalOriginMessages(0)
- , OriginMessagesCounter(0)
- , TotalProcessedMessages(0)
- , ProcessedMessagesCounter(0)
- {}
-
- bool IsReadyForCommit() {
- return TotalOriginMessages == OriginMessagesCounter && TotalProcessedMessages == ProcessedMessagesCounter;
- }
-};
-
-struct TProducerKey {
- TString Topic;
- TString SourceId;
-
- bool operator <(const TProducerKey& other) const {
- return std::tie(Topic, SourceId) < std::tie(other.Topic, other.SourceId);
- }
-};
-
-struct TProduceInfo {
- NThreading::TFuture<TProducerCommitResponse> Ack;
- TDataInfo* DataInfo = nullptr;
-
- explicit TProduceInfo(const NThreading::TFuture<TProducerCommitResponse>& ack, TDataInfo* dataInfo)
- : Ack(ack)
- , DataInfo(dataInfo)
- {}
-};
-
-struct TProducerInfo {
- std::shared_ptr<IProducerImpl> Producer;
- TDeque<TProduceInfo> Queue;
- TInstant LastWriteTime;
-};
-
-struct TProcessingData {
- NThreading::TFuture<TProcessedData> Processed;
- ui64 OriginMessageOffset;
- ui64 OriginMessageSize;
- TDataInfo* DataInfo;
-
- TProcessingData(const NThreading::TFuture<TProcessedData>& processed, ui64 originMessageOffset, ui64 originMessageSize, TDataInfo* dataInfo)
- : Processed(processed)
- , OriginMessageOffset(originMessageOffset)
- , OriginMessageSize(originMessageSize)
- , DataInfo(dataInfo)
- {};
-};
-
-class TPQLibPrivate;
-
-class TProcessor: public IProcessorImpl, public std::enable_shared_from_this<TProcessor> {
-public:
- TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept;
- ~TProcessor() noexcept;
- void Init() override;
-
- using IProcessorImpl::GetNextData;
- void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept override;
-
- NThreading::TFuture<void> Destroyed() noexcept override;
-
- void Cancel() override;
-
-protected:
- TProcessorSettings Settings;
- TIntrusivePtr<ILogger> Logger;
- std::shared_ptr<IConsumerImpl> Consumer;
- NThreading::TFuture<TConsumerCreateResponse> ConsumerIsStarted;
- NThreading::TFuture<TError> ConsumerIsDead;
- NThreading::TFuture<TConsumerMessage> ConsumerRequest;
- TMap<TProducerKey, TProducerInfo> Producers;
- ui64 CurrentOriginDataMemoryUsage;
- ui64 CurrentProcessedDataMemoryUsage;
-
- TDeque<NThreading::TPromise<TOriginData>> Requests;
- TDeque<TDataInfo> Data;
- TMap<TPartition, TDeque<TProcessingData>> ProcessingData;
- NThreading::TPromise<void> ObjectsDestroyed = NThreading::NewPromise<void>();
- bool DestroyedFlag = false;
-
- template<typename T>
- void SafeSubscribe(NThreading::TFuture<T>& future);
- void CleanState() noexcept;
- bool NextRequest() noexcept;
- void RecreateConsumer() noexcept;
- void ScheduleIdleSessionsCleanup() noexcept;
- void CloseIdleSessions() noexcept;
- void ProcessFutures() noexcept;
- bool ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept;
- void SubscribeDestroyed();
- void InitImpl();
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp
deleted file mode 100644
index d36573dcfc..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp
+++ /dev/null
@@ -1,133 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
-namespace NPersQueue {
-Y_UNIT_TEST_SUITE(TProcessorTest) {
- TConsumerSettings MakeConsumerSettings(const TString& clientId, const TVector<TString>& topics,
- const TTestServer& testServer) {
- TConsumerSettings consumerSettings;
- consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- consumerSettings.ClientId = clientId;
- consumerSettings.Topics = topics;
- consumerSettings.UseLockSession = false;
- return consumerSettings;
- }
-
- TProducerSettings MakeProducerSettings(const TString& topic, const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = true;
- producerSettings.Topic = topic;
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::RAW;
- return producerSettings;
- }
-
- TProcessorSettings MakeProcessorSettings(const TTestServer& testServer) {
- TProcessorSettings processorSettings;
- processorSettings.ConsumerSettings = MakeConsumerSettings("processor", {"input"}, testServer);
- processorSettings.ProducerSettings = MakeProducerSettings("output", testServer);
- return processorSettings;
- }
-
- void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response);
- }
-
- void ProcessMessage(TOriginData& originData) {
- for (auto& messages: originData.Messages) {
- for (auto& message: messages.second) {
- auto msg = message.Message;
-
- TVector<TProcessedMessage> processedMessages;
- for (int i = 0; i < 3; ++i) {
- TStringBuilder sourceIdPrefix;
- sourceIdPrefix << "shard" << i << "_";
- processedMessages.push_back(TProcessedMessage {"output", ToString(msg.GetOffset()), sourceIdPrefix, 0});
- }
- message.Processed.SetValue(TProcessedData{processedMessages});
- }
- }
- }
-
- bool IsFinished(const TTestServer& testServer) {
- auto clientInfo = testServer.AnnoyingClient->GetClientInfo({"rt3.dc1--input"}, "processor", true);
- auto topicResult = clientInfo.GetMetaResponse().GetCmdGetReadSessionsInfoResult().GetTopicResult(0).GetPartitionResult();
- for (auto& partition: topicResult) {
- if (partition.GetClientOffset() < partition.GetEndOffset()) return false;
- }
- return true;
- }
-
- void BaseFunctionalityTest() {
- TTestServer testServer(false);
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--input", 2);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--output", 4);
-
- auto producerSettings = MakeProducerSettings("input", testServer);
- auto producer = testServer.PQLib->CreateProducer(producerSettings, testServer.PQLibSettings.DefaultLogger, false);
- producer->Start();
-
- for (int i = 1; i < 4; ++i) {
- auto write1 = producer->Write(i, TString("blob"));
- AssertWriteValid(write1);
- }
-
- auto processorSettings = MakeProcessorSettings(testServer);
- auto processor = testServer.PQLib->CreateProcessor(processorSettings, testServer.PQLibSettings.DefaultLogger, false);
-
- while (!IsFinished(testServer)) {
- auto data = processor->GetNextData();
- data.Subscribe([](const auto& f) {
- auto value = f.GetValueSync();
- ProcessMessage(value);
- });
- Sleep(TDuration::MilliSeconds(100));
- }
-
- auto consumerSettings = MakeConsumerSettings("checker", {"output"}, testServer);
- auto consumer = testServer.PQLib->CreateConsumer(consumerSettings, nullptr, false);
- auto isStarted = consumer->Start().ExtractValueSync();
- UNIT_ASSERT_C(isStarted.Response.HasInit(), "cannot start consumer " << isStarted.Response);
-
- ui32 totalMessages = 9;
- ui32 currentMessages = 0;
- THashMap<TString, ui32> expectedSeqno = {
- {"shard0_rt3.dc1--input:0_output", 1}, {"shard1_rt3.dc1--input:0_output", 1}, {"shard2_rt3.dc1--input:0_output", 1},
- {"shard0_rt3.dc1--input:1_output", 1}, {"shard1_rt3.dc1--input:1_output", 1}, {"shard2_rt3.dc1--input:1_output", 1},
- {"shard0_rt3.dc1--input:2_output", 1}, {"shard1_rt3.dc1--input:2_output", 1}, {"shard2_rt3.dc1--input:2_output", 1},
- {"shard0_rt3.dc1--input:3_output", 1}, {"shard1_rt3.dc1--input:3_output", 1}, {"shard2_rt3.dc1--input:3_output", 1},
- };
- while (currentMessages < totalMessages) {
- auto f = consumer->GetNextMessage();
- for (auto& batch: f.GetValueSync().Response.GetData().GetMessageBatch()) {
- for (auto& message: batch.GetMessage()) {
- TString sourceId = message.GetMeta().GetSourceId();
- ui32 seqNo = message.GetMeta().GetSeqNo();
- UNIT_ASSERT_EQUAL(ToString(seqNo - 1), message.GetData());
- auto it = expectedSeqno.find(sourceId);
- UNIT_ASSERT_UNEQUAL_C(it, expectedSeqno.end(), "unknown sourceId " << sourceId);
- UNIT_ASSERT_EQUAL(it->second, seqNo);
- it->second++;
- currentMessages += 1;
- }
- }
- }
- }
-
- Y_UNIT_TEST(BasicFunctionality) {
- BaseFunctionalityTest();
- }
-}
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp
deleted file mode 100644
index 2de983453f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp
+++ /dev/null
@@ -1,506 +0,0 @@
-#include <util/generic/strbuf.h>
-#include <util/stream/zlib.h>
-#include <util/string/cast.h>
-#include <util/string/printf.h>
-#include <util/string/vector.h>
-#include <util/string/builder.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h>
-
-#include <grpc++/create_channel.h>
-
-
-namespace NPersQueue {
-
-class TProducerDestroyHandler : public IHandler {
-public:
- TProducerDestroyHandler(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : Ptr(std::move(ptr))
- , RpcStuff(std::move(rpcStuff))
- , QueueTag(queueTag)
- , PQLib(pqLib)
- {}
-
- void Destroy(const TError&) override {
- auto producer = Ptr;
- auto handler = [producer] {
- auto producerShared = producer.lock();
- if (producerShared) {
- producerShared->OnFail();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "TProducerDestroyHandler"; }
-
-protected:
- std::weak_ptr<TProducer> Ptr;
- TIntrusivePtr<TProducer::TRpcStuff> RpcStuff; // must simply live
- const void* QueueTag;
- TPQLibPrivate* PQLib;
-};
-
-class TStreamCreated : public TProducerDestroyHandler {
-public:
- TStreamCreated(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto producer = Ptr;
- auto UA = PQLib->GetUserAgent();
- auto handler = [producer, UA] {
- auto producerShared = producer.lock();
- if (producerShared) {
- producerShared->OnStreamCreated(UA);
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- TString ToString() override { return "StreamCreated"; }
-};
-
-class TFinishDone : public TProducerDestroyHandler {
-public:
- TFinishDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto producer = Ptr;
- auto handler = [producer] {
- auto producerShared = producer.lock();
- if (producerShared) {
- producerShared->OnFinishDone();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
-
- void Destroy(const TError&) override {
- Y_FAIL("Finish call failed");
- }
-
- TString ToString() override { return "Finish"; }
-};
-
-
-class TWriteDone : public TProducerDestroyHandler {
-public:
- TWriteDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto producer = Ptr;
- auto handler = [producer] {
- auto producerShared = producer.lock();
- if (producerShared) {
- producerShared->OnWriteDone();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
- TString ToString() override { return "WriteDone"; }
-};
-
-
-class TReadDone : public TProducerDestroyHandler {
-public:
- TReadDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff,
- const void* queueTag, TPQLibPrivate* pqLib)
- : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib)
- {}
-
- void Done() override {
- auto producer = Ptr;
- auto handler = [producer] {
- auto producerShared = producer.lock();
- if (producerShared) {
- producerShared->OnReadDone();
- }
- };
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler));
- }
- TString ToString() override { return "ReadDone"; }
-};
-
-TProducer::TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq,
- NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept
- : IProducerImpl(std::move(destroyEventRef), std::move(pqLib))
- , RpcStuff(new TRpcStuff())
- , Settings(settings)
- , StartPromise(promise)
- , IsDeadPromise(NThreading::NewPromise<TError>())
- , Pos(0)
- , WriteInflight(false)
- , ProxyCookie(0)
- , Logger(std::move(logger))
- , IsDestroyed(false)
- , IsDestroying(false)
- , Failing(false)
-{
- RpcStuff->CQ = std::move(cq);
-}
-
-TProducer::~TProducer() {
- Destroy();
-}
-
-void TProducer::Destroy() noexcept {
- IsDestroying = true;
- RpcStuff->Context.TryCancel();
- ChannelHolder.ChannelPtr = nullptr;
-
- Destroy("Destructor called");
-}
-
-void TProducer::DoStart(TInstant deadline) {
- if (IsDestroyed) {
- return;
- }
-
- if (deadline != TInstant::Max()) {
- std::weak_ptr<TProducer> self = shared_from_this();
- auto onStartTimeout = [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnStartTimeout();
- }
- };
- StartDeadlineCallback =
- PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout);
- }
-
- FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get());
- RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel);
- RpcStuff->Stream = RpcStuff->Stub->AsyncWriteSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler));
-}
-
-void TProducer::Init() {
- std::weak_ptr<TProducer> self = shared_from_this(); // we can't make this object in constructor, because this will be the only reference to us and ~shared_ptr() will destroy us.
- StreamCreatedHandler.Reset(new TStreamCreated(self, RpcStuff, this, PQLib.Get()));
- ReadDoneHandler.Reset(new TReadDone(self, RpcStuff, this, PQLib.Get()));
- WriteDoneHandler.Reset(new TWriteDone(self, RpcStuff, this, PQLib.Get()));
- FinishDoneHandler.Reset(new TFinishDone(self, RpcStuff, this, PQLib.Get()));
-}
-
-NThreading::TFuture<TProducerCreateResponse> TProducer::Start(TInstant deadline) noexcept {
- if (IsDestroyed) {
- TWriteResponse res;
- res.MutableError()->SetDescription("Producer is dead");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- return NThreading::MakeFuture<TProducerCreateResponse>(TProducerCreateResponse{std::move(res)});
- }
- NThreading::TFuture<TProducerCreateResponse> ret = StartPromise.GetFuture();
- if (ChannelHolder.ChannelInfo.Initialized()) {
- std::weak_ptr<TProducer> self = shared_from_this();
- PQLib->Subscribe(ChannelHolder.ChannelInfo,
- this,
- [self, deadline](const auto& f) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->SetChannel(f.GetValue());
- selfShared->DoStart(deadline);
- }
- });
- } else {
- if (RpcStuff->Channel && ProxyCookie) {
- DoStart(deadline);
- } else {
- Destroy("No channel");
- }
- }
- return ret;
-}
-
-void TProducer::SetChannel(const TChannelHolder& channel) noexcept {
- if (IsDestroyed) {
- return;
- }
-
- ChannelHolder = channel;
-}
-
-void TProducer::SetChannel(const TChannelInfo& channel) noexcept {
- if (IsDestroyed) {
- return;
- }
-
- Y_VERIFY(!RpcStuff->Channel);
- if (!channel.Channel || channel.ProxyCookie == 0) {
- if (channel.Channel) {
- Destroy("ProxyCookie is zero");
- } else {
- Destroy("Channel creation error");
- }
-
- return;
- }
-
- RpcStuff->Channel = channel.Channel;
- ProxyCookie = channel.ProxyCookie;
-}
-
-
-static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) {
- TWriteResponse res;
- res.MutableError()->SetDescription(description);
- res.MutableError()->SetCode(NErrorCode::ERROR);
- return TProducerCommitResponse{seqNo, data, std::move(res)};
-}
-
-void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept {
- return TProducer::Write(promise, MaxSeqNo + 1, std::move(data));
-}
-
-void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept {
- Y_VERIFY(data.IsEncoded());
-
- MaxSeqNo = Max(seqNo, MaxSeqNo);
-
- if (IsDestroyed) {
- promise.SetValue(MakeErrorCommitResponse("producer is dead", seqNo, data));
- return;
- }
-
- if (StartPromise.Initialized()) {
- promise.SetValue(MakeErrorCommitResponse("producer is not ready", seqNo, data));
- return;
- }
-
- CommitPromises.push_back(promise);
- Data.emplace_back(seqNo, std::move(data));
- DoWrite();
-}
-
-void TProducer::OnWriteDone() {
- if (IsDestroyed) {
- return;
- }
-
- WriteInflight = false;
- DoWrite();
-}
-
-void TProducer::DoWrite() {
- if (Failing && !WriteInflight) {
- return;
- }
- if (WriteInflight || Pos == Data.size()) {
- return;
- }
- WriteInflight = true;
- TWriteRequest req;
- Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials());
-
- req.MutableData()->SetSeqNo(Data[Pos].SeqNo);
- req.MutableData()->SetData(Data[Pos].Data.GetEncodedData());
- req.MutableData()->SetCodec(Data[Pos].Data.GetCodecType());
- req.MutableData()->SetCreateTimeMs(Data[Pos].Data.GetTimestamp().MilliSeconds());
-
- ++Pos;
- RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler));
-}
-
-NThreading::TFuture<TError> TProducer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TProducer::Destroy(const TString& description) {
- TError error;
- error.SetDescription(description);
- error.SetCode(NErrorCode::ERROR);
- Destroy(error);
-}
-
-void TProducer::Destroy(const TError& error) {
- if (IsDestroyed) {
- return;
- }
-
- if (!IsDestroying) {
- INFO_LOG("Error: " << error, Settings.SourceId, SessionId);
- }
-
- IsDestroyed = true;
- IsDeadPromise.SetValue(error);
-
- if (StartDeadlineCallback) {
- StartDeadlineCallback->TryCancel();
- }
-
- NThreading::TFuture<TChannelInfo> tmp;
- tmp.Swap(ChannelHolder.ChannelInfo);
- ChannelHolder.ChannelPtr = nullptr;
-
- Error = error;
- if (StartPromise.Initialized()) {
- NThreading::TPromise<TProducerCreateResponse> tmp;
- tmp.Swap(StartPromise);
- TWriteResponse res;
- res.MutableError()->CopyFrom(Error);
- tmp.SetValue(TProducerCreateResponse{std::move(res)});
- }
- while (!CommitPromises.empty()) {
- auto p = CommitPromises.front();
- CommitPromises.pop_front();
- auto pp(std::move(Data.front()));
- Data.pop_front();
- if (Pos > 0) {
- --Pos;
- }
- TWriteResponse res;
- res.MutableError()->CopyFrom(Error);
-
- p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)});
- }
-
- StreamCreatedHandler.Reset();
- ReadDoneHandler.Reset();
- WriteDoneHandler.Reset();
- FinishDoneHandler.Reset();
-
- DestroyPQLibRef();
-}
-
-void TProducer::OnStreamCreated(const TString& userAgent) {
- if (IsDestroyed) {
- return;
- }
-
- TWriteRequest req;
- Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials());
-
- req.MutableInit()->SetSourceId(Settings.SourceId);
- req.MutableInit()->SetTopic(Settings.Topic);
- req.MutableInit()->SetProxyCookie(ProxyCookie);
- req.MutableInit()->SetPartitionGroup(Settings.PartitionGroup);
- req.MutableInit()->SetVersion(userAgent);
- Y_VERIFY(!userAgent.empty());
-
- for (const auto& attr : Settings.ExtraAttrs) {
- auto item = req.MutableInit()->MutableExtraFields()->AddItems();
- item->SetKey(attr.first);
- item->SetValue(attr.second);
- }
-
- WriteInflight = true;
- RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler));
-
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-}
-
-void TProducer::OnFail() {
- if (Failing)
- return;
- Failing = true;
-
- if (IsDestroyed) {
- return;
- }
-
- RpcStuff->Stream->Finish(&RpcStuff->Status, new TQueueEvent(FinishDoneHandler));
-}
-
-void TProducer::OnFinishDone() {
- if (IsDestroyed) {
- return;
- }
-
- TError error;
- const auto& msg = RpcStuff->Status.error_message();
- TString reason(msg.data(), msg.length());
- error.SetDescription(reason);
- error.SetCode(NErrorCode::ERROR);
-
- Destroy(error);
-}
-
-void TProducer::OnReadDone() {
- if (IsDestroyed) {
- return;
- }
-
- if (RpcStuff->Response.HasError()) {
- Destroy(RpcStuff->Response.GetError());
- return;
- }
-
- if (StartPromise.Initialized()) { //init response
- NThreading::TPromise<TProducerCreateResponse> tmp;
- tmp.Swap(StartPromise);
- Y_VERIFY(RpcStuff->Response.HasInit());
- auto res(std::move(RpcStuff->Response));
- RpcStuff->Response.Clear();
-
- SessionId = res.GetInit().GetSessionId();
- const ui32 partition = res.GetInit().GetPartition();
- MaxSeqNo = res.GetInit().GetMaxSeqNo();
- const TProducerSeqNo seqNo = MaxSeqNo;
- const TString topic = res.GetInit().GetTopic();
- tmp.SetValue(TProducerCreateResponse{std::move(res)});
-
- if (StartDeadlineCallback) {
- StartDeadlineCallback->TryCancel();
- }
- StartDeadlineCallback = nullptr;
-
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-
- DEBUG_LOG("Stream created to topic " << topic << " partition " << partition << " maxSeqNo " << seqNo, Settings.SourceId, SessionId);
- } else { //write response
- //get first CommitPromise
- Y_VERIFY(!Data.empty());
- auto p = CommitPromises.front();
- CommitPromises.pop_front();
- auto pp(std::move(Data.front()));
- Data.pop_front();
- Y_VERIFY(Pos > 0);
- --Pos;
-
- Y_VERIFY(RpcStuff->Response.HasAck());
- Y_VERIFY(RpcStuff->Response.GetAck().GetSeqNo() == pp.SeqNo);
- auto res(std::move(RpcStuff->Response));
- RpcStuff->Response.Clear();
- RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler));
-
- p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)});
- }
-}
-
-void TProducer::OnStartTimeout() {
- if (IsDestroyed) {
- return;
- }
-
- StartDeadlineCallback = nullptr;
- if (!StartPromise.Initialized()) {
- // everything is OK, there is no timeout, we have already started.
- return;
- }
-
- TError error;
- error.SetDescription("Start timeout");
- error.SetCode(NErrorCode::CREATE_TIMEOUT);
- Destroy(error);
-}
-
-void TProducer::Cancel() {
- IsDestroying = true;
- RpcStuff->Context.TryCancel();
- ChannelHolder.ChannelPtr = nullptr;
-
- Destroy(GetCancelReason());
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h
deleted file mode 100644
index af74ced4b9..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h
+++ /dev/null
@@ -1,107 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h>
-#include <library/cpp/threading/future/future.h>
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-#include <deque>
-
-namespace NPersQueue {
-
-class TProducer : public IProducerImpl, public std::enable_shared_from_this<TProducer> {
-public:
- using IProducerImpl::Write;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override;
-
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override;
-
- TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq,
- NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept;
- ~TProducer();
-
- void Destroy() noexcept;
- void SetChannel(const TChannelHolder& channel) noexcept;
- void SetChannel(const TChannelInfo& channel) noexcept;
-
- void DoStart(TInstant deadline);
-
- void Init() override;
-
- const TProducerSettings& GetSettings() const {
- return Settings;
- }
-
- void Cancel() override;
-
-public:
- using TStream = grpc::ClientAsyncReaderWriterInterface<TWriteRequest, TWriteResponse>;
-
- // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue
- struct TRpcStuff: public TAtomicRefCount<TRpcStuff> {
- TWriteResponse Response;
- std::shared_ptr<grpc::CompletionQueue> CQ;
- std::shared_ptr<grpc::Channel> Channel;
- std::unique_ptr<PersQueueService::Stub> Stub;
- grpc::ClientContext Context;
- std::unique_ptr<TStream> Stream;
- grpc::Status Status;
- };
-
-private:
- friend class TPQLibPrivate;
- friend class TStreamCreated;
- friend class TReadDone;
- friend class TWriteDone;
- friend class TProducerDestroyHandler;
- friend class TFinishDone;
-
- IHandlerPtr StreamCreatedHandler;
- IHandlerPtr ReadDoneHandler;
- IHandlerPtr WriteDoneHandler;
- IHandlerPtr FinishDoneHandler;
-
- void Destroy(const TError& error);
- void Destroy(const TString& description); // the same but with Code=ERROR
- void OnStreamCreated(const TString& userAgent);
- void OnReadDone();
- void OnWriteDone();
- void DoWrite();
- void OnFail();
- void OnFinishDone();
- void OnStartTimeout();
-
-protected:
- TIntrusivePtr<TRpcStuff> RpcStuff;
-
- TChannelHolder ChannelHolder;
- TProducerSettings Settings;
-
- TString SessionId;
-
- NThreading::TPromise<TProducerCreateResponse> StartPromise;
- NThreading::TPromise<TError> IsDeadPromise;
- std::deque<NThreading::TPromise<TProducerCommitResponse>> CommitPromises;
- std::deque<TWriteData> Data;
- ui32 Pos = 0;
- bool WriteInflight;
- ui64 ProxyCookie = 0;
- TProducerSeqNo MaxSeqNo = 0;
-
- TIntrusivePtr<ILogger> Logger;
-
- TError Error;
-
- bool IsDestroyed;
- bool IsDestroying;
- bool Failing;
- TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp
deleted file mode 100644
index 5f3deb880d..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp
+++ /dev/null
@@ -1,202 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NPersQueue {
-
-Y_UNIT_TEST_SUITE(TProducerTest) {
- Y_UNIT_TEST(NotStartedProducerCanBeDestructed) {
- // Test that producer doesn't hang on till shutdown
- TPQLib lib;
- TProducerSettings settings;
- settings.Server = TServerSetting{"localhost"};
- settings.Topic = "topic";
- settings.SourceId = "src";
- lib.CreateProducer(settings, {}, false);
- }
-
- TProducerSettings MakeProducerSettings(const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = false;
- producerSettings.Topic = "topic1";
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::LZOP;
- return producerSettings;
- }
-
- Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- const size_t partitions = 1;
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
-
- testServer.WaitInit("topic1");
-
- auto producer = testServer.PQLib->CreateProducer(MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false);
- UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError());
- auto isDead = producer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto write1 = producer->Write(1, TString("blob1"));
- auto write2 = producer->Write(2, TString("blob2"));
- auto write3 = producer->Write(3, TString("blob3"));
- auto write4 = producer->Write(4, TString("blob4"));
- auto write5 = producer->Write(5, TString("blob5"));
-
- testServer.PQLib = nullptr;
- Cerr << "PQLib destroyed" << Endl;
-
- UNIT_ASSERT(write1.HasValue());
- UNIT_ASSERT(write2.HasValue());
- UNIT_ASSERT(write3.HasValue());
- UNIT_ASSERT(write4.HasValue());
- UNIT_ASSERT(write5.HasValue());
-
- UNIT_ASSERT(write1.GetValue().Response.HasError());
- UNIT_ASSERT(write2.GetValue().Response.HasError());
- UNIT_ASSERT(write3.GetValue().Response.HasError());
- UNIT_ASSERT(write4.GetValue().Response.HasError());
- UNIT_ASSERT(write5.GetValue().Response.HasError());
-
- auto write6 = producer->Write(6, TString("blob6"));
- UNIT_ASSERT(write6.HasValue());
- UNIT_ASSERT(write6.GetValue().Response.HasError());
- }
-
- Y_UNIT_TEST(WriteToDeadProducer) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false);
- auto f = producer->Start();
- UNIT_ASSERT(f.GetValueSync().Response.HasError());
- Cerr << f.GetValueSync().Response << "\n";
- auto isDead = producer->IsDead();
- isDead.Wait();
- UNIT_ASSERT(isDead.HasValue());
-
- auto write = producer->Write(1, TString("blob"));
-
- Cerr << write.GetValueSync().Response << "\n";
- UNIT_ASSERT(write.GetValueSync().Response.HasError());
- UNIT_ASSERT_STRINGS_EQUAL(write.GetValueSync().Response.GetError().GetDescription(), "Destroyed");
- }
-
- Y_UNIT_TEST(Auth_StartProducerWithInvalidTokenFromGrpcMetadataPointOfView_StartFailedAndProduerDiedWithBadRequestErrors) {
- TPQLibSettings pqLibSettings;
- if (!std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"))
- return;
-
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer();
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto fillCredentialsWithInvalidToken = [](NPersQueue::TCredentials* authInfo) {
- authInfo->set_tvm_service_ticket("token\n");
- };
- auto producerSettings = MakeProducerSettings(testServer);
- producerSettings.CredentialsProvider = std::make_shared<TCallbackCredentialsProvider>(std::move(fillCredentialsWithInvalidToken));
- auto producer = PQLib->CreateProducer(producerSettings, logger, false);
-
-
- Cerr << "Wait for producer start procedure end" << Endl;
- auto startResponse = producer->Start().GetValueSync().Response;
-
- UNIT_ASSERT(startResponse.has_error());
- UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, startResponse.error().code(), startResponse);
-
- Cerr << "Wait for producer death" << Endl;
- auto deathCause = producer->IsDead().GetValueSync();
-
-
- UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, deathCause.code(), deathCause);
- }
-
- Y_UNIT_TEST(Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerDiesWithBadRequestError) {
- SDKTestSetup setup{"Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerIsDeadWithBadRequestError"};
- auto log = setup.GetLog();
- auto producerSettings = setup.GetProducerSettings();
- // TTestServer::AnnoyingClient creates topic with default codecs set: raw, gzip, lzop. zstd not included
- producerSettings.Codec = ECodec::ZSTD;
- auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false);
- log << TLOG_INFO << "Wait for producer start";
- auto startResponse = producer->Start().GetValueSync().Response;
- UNIT_ASSERT_C(!startResponse.HasError(), startResponse);
-
- log << TLOG_INFO << "Wait for write response";
- auto writeResponse = producer->Write(NUnitTest::RandomString(250 * 1024, std::rand())).GetValueSync().Response;
-
-
- UNIT_ASSERT(writeResponse.HasError());
- Cerr << writeResponse << "\n";
- UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::BAD_REQUEST, writeResponse.GetError().GetCode(), writeResponse);
- }
-
- void StartProducerWithDiscovery(bool cds, const TString& preferredCluster = {}, bool addBrokenDatacenter = !GrpcV1EnabledByDefault()) {
- SDKTestSetup setup{"StartProducerWithDiscovery", false};
- setup.Start(true, addBrokenDatacenter);
- auto log = setup.GetLog();
- auto producerSettings = setup.GetProducerSettings();
- if (preferredCluster) {
- producerSettings.PreferredCluster = preferredCluster;
- }
- auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false);
- auto startResponse = producer->Start().GetValueSync().Response;
- UNIT_ASSERT_C(!startResponse.HasError(), startResponse);
-
- log << TLOG_INFO << "Wait for write response";
- auto writeResponse = producer->Write(1, TString("blob1")).GetValueSync().Response;
- UNIT_ASSERT(!writeResponse.HasError());
-
- producerSettings.Server.UseLogbrokerCDS = cds ? EClusterDiscoveryUsageMode::Use : EClusterDiscoveryUsageMode::DontUse;
- producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false);
- startResponse = producer->Start().GetValueSync().Response;
- UNIT_ASSERT_C(!startResponse.HasError(), startResponse);
-
- log << TLOG_INFO << "Wait for write response";
- writeResponse = producer->Write(2, TString("blob2")).GetValueSync().Response;
- UNIT_ASSERT(!writeResponse.HasError());
- }
-
- Y_UNIT_TEST(StartProducerWithCDS) {
- StartProducerWithDiscovery(true);
- }
-
- Y_UNIT_TEST(StartProducerWithoutCDS) {
- StartProducerWithDiscovery(false);
- }
-
- Y_UNIT_TEST(StartProducerWithCDSAndPreferAvailableCluster) {
- StartProducerWithDiscovery(true, "dc1");
- }
-
- Y_UNIT_TEST(StartProducerWithCDSAndPreferUnavailableCluster) {
- StartProducerWithDiscovery(true, "dc2", true);
- }
-
- Y_UNIT_TEST(StartProducerWithCDSAndPreferUnknownCluster) {
- SDKTestSetup setup{"PreferUnknownCluster"};
- auto producerSettings = setup.GetProducerSettings();
- producerSettings.Server.UseLogbrokerCDS = EClusterDiscoveryUsageMode::Use;
- producerSettings.PreferredCluster = "blablabla";
- auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false);
- auto startResponse = producer->Start().GetValueSync().Response;
- UNIT_ASSERT_C(startResponse.HasError(), startResponse);
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp
deleted file mode 100644
index 6e1c3f9b2e..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-#include "queue_pool.h"
-
-#include <util/digest/city.h>
-
-namespace NPersQueue {
-void TQueuePool::Start(size_t queuesCount) {
- Y_VERIFY(queuesCount > 0);
- Queues.resize(queuesCount);
- for (auto& queuePtr : Queues) {
- queuePtr = std::make_shared<TThreadPool>();
- queuePtr->Start(1); // start one thread for each tag
- }
-}
-
-void TQueuePool::Stop() {
- for (auto& queuePtr : Queues) {
- queuePtr->Stop();
- }
- Queues.clear();
-}
-
-const std::shared_ptr<IThreadPool>& TQueuePool::GetQueuePtr(const void* tag) {
- Y_VERIFY(!Queues.empty());
- const size_t queue = static_cast<size_t>(CityHash64(reinterpret_cast<const char*>(&tag), sizeof(tag))) % Queues.size();
- return Queues[queue];
-}
-
-IThreadPool& TQueuePool::GetQueue(const void* tag) {
- return *GetQueuePtr(tag);
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h
deleted file mode 100644
index 961b8ff446..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-#include <util/generic/ptr.h>
-#include <util/thread/pool.h>
-
-#include <functional>
-#include <memory>
-#include <vector>
-
-namespace NPersQueue {
-class TQueuePool {
-public:
- void Start(size_t queuesCount);
- void Stop();
-
- // get one-thread-queue for processing specified tag (address)
- IThreadPool& GetQueue(const void* tag);
-
- const std::shared_ptr<IThreadPool>& GetQueuePtr(const void* tag);
-
-private:
- std::vector<std::shared_ptr<IThreadPool>> Queues;
-};
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp
deleted file mode 100644
index 9f8d792e52..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#include "queue_pool.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <unordered_map>
-
-namespace NPersQueue {
-Y_UNIT_TEST_SUITE(TQueuePoolTest) {
- Y_UNIT_TEST(QueuesDistribution) {
- TQueuePool pool;
- pool.Start(10);
- size_t addresses[10000] = {};
- std::unordered_map<IThreadPool*, size_t> queueAddresses;
- for (size_t& address : addresses) {
- IThreadPool* q = &pool.GetQueue(&address);
- ++queueAddresses[q];
- UNIT_ASSERT_EQUAL(q, &pool.GetQueue(&address)); // one address always leads to one queue
- }
- UNIT_ASSERT_VALUES_EQUAL(queueAddresses.size(), 10);
- for (const auto& queueToCount : queueAddresses) {
- UNIT_ASSERT_C(queueToCount.second >= 850, "Count: " << queueToCount.second);
- UNIT_ASSERT_C(queueToCount.second <= 1150, "Count: " << queueToCount.second);
- }
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp
deleted file mode 100644
index 02b3aafafc..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp
+++ /dev/null
@@ -1,526 +0,0 @@
-#include "retrying_consumer.h"
-
-namespace {
- bool IsRetryable(const NPersQueue::TError& err) {
- switch (err.code()) {
- case NPersQueue::NErrorCode::INITIALIZING:
- case NPersQueue::NErrorCode::OVERLOAD:
- case NPersQueue::NErrorCode::READ_TIMEOUT:
- case NPersQueue::NErrorCode::TABLET_IS_DROPPED:
- case NPersQueue::NErrorCode::CREATE_TIMEOUT:
- case NPersQueue::NErrorCode::ERROR:
- case NPersQueue::NErrorCode::CLUSTER_DISABLED:
- return true;
- default:
- return false;
- }
- }
-}
-
-namespace NPersQueue {
-
-TRetryingConsumer::TRetryingConsumer(const TConsumerSettings& settings,
- std::shared_ptr<void> destroyEventRef,
- TIntrusivePtr<TPQLibPrivate> pqLib,
- TIntrusivePtr<ILogger> logger)
- : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , Logger(std::move(logger))
- , IsDeadPromise(NThreading::NewPromise<TError>())
- , ConsumerDestroyedPromise(NThreading::NewPromise<void>())
- , GenCounter(1)
- , CookieCounter(1)
- , ReconnectionAttemptsDone(0)
- , Stopping(false)
- , Reconnecting(false)
-{
- Y_VERIFY(Settings.ReconnectOnFailure, "ReconnectOnFailure should be set.");
- Y_ENSURE(Settings.MaxAttempts != 0, "MaxAttempts setting can't be zero.");
- Y_ENSURE(Settings.ReconnectionDelay != TDuration::Zero(), "ReconnectionDelay setting can't be zero.");
- Y_ENSURE(Settings.MaxReconnectionDelay != TDuration::Zero(), "MaxReconnectionDelay setting can't be zero.");
- Y_ENSURE(Settings.StartSessionTimeout != TDuration::Zero(), "StartSessionTimeout setting can't be zero.");
-
- Settings.ReconnectOnFailure = false; // reset flag, to use the settings for creating sub consumer
-}
-
-TRetryingConsumer::~TRetryingConsumer() noexcept {
- Cancel();
-}
-
-NThreading::TFuture<TConsumerCreateResponse> TRetryingConsumer::Start(TInstant deadline) noexcept {
- StartPromise = NThreading::NewPromise<TConsumerCreateResponse>();
- if (deadline != TInstant::Max()) {
- std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this();
- auto onDeadline = [weakRef] {
- auto self = weakRef.lock();
- if (nullptr != self) {
- self->OnStartDeadline();
- }
- };
- PQLib->GetScheduler().Schedule(deadline, this, onDeadline);
- }
- DoReconnect(deadline);
- return StartPromise.GetFuture();
-}
-
-NThreading::TFuture<TError> TRetryingConsumer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TRetryingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept {
- if (!StartPromise.Initialized() || Stopping) {
- TReadResponse res;
- res.MutableError()->SetDescription("consumer is not ready");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- promise.SetValue(TConsumerMessage{std::move(res)});
- return;
- }
-
- if (!ReadyResponses.empty()) {
- Y_ASSERT(PendingRequests.empty());
- promise.SetValue(std::move(ReadyResponses.front()));
- ReadyResponses.pop_front();
- return;
- }
-
- PendingRequests.push_back(promise);
-
- if (Consumer && StartFuture.HasValue()) {
- DoRequest();
- }
-}
-
-static void FormatCookies(TStringBuilder& ret, const TVector<ui64>& cookies) {
- ret << "{";
- for (size_t i = 0; i < cookies.size(); ++i) {
- if (i > 0) {
- ret << ", ";
- }
- ret << cookies[i];
- }
- ret << "}";
-}
-
-static TString FormatCommitForLog(const TStringBuf reason, const TVector<ui64>& cookies, const TVector<ui64>& originalCookies, const TVector<ui64>& committedCookies) {
- TStringBuilder ret;
- ret << "Commit cookies by retrying consumer" << reason << ". User cookies: ";
- FormatCookies(ret, cookies);
- ret << ". Subconsumer cookies to commit: ";
- FormatCookies(ret, originalCookies);
- ret << ". Skipped cookies: ";
- FormatCookies(ret, committedCookies);
- return std::move(ret);
-}
-
-void TRetryingConsumer::Commit(const TVector<ui64>& cookies) noexcept {
- if (!Consumer) {
- if (!StartPromise.Initialized() || Stopping) {
- Destroy("Not ready", NErrorCode::BAD_REQUEST);
- } else {
- // just response that cookies were commited
- FastCommit(cookies);
- DEBUG_LOG(FormatCommitForLog(". Consumer is not initialied", cookies, {}, cookies), "", SessionId);
- }
- return;
- }
- if (Settings.CommitsDisabled) {
- Destroy("Commits are disabled", NErrorCode::BAD_REQUEST);
- return;
- }
-
- TVector<ui64> originalCookies(Reserve(cookies.size()));
- // cookies which can be treated as committed
- TVector<ui64> commitedCookies;
- ui64 minCookie = CookieCounter - Cookies.size();
- for (auto cookie : cookies) {
- if (cookie >= minCookie && cookie < CookieCounter) {
- auto& cookieInfo = Cookies[cookie - minCookie];
- Y_VERIFY(cookieInfo.UserCookie == cookie);
- if (0 != cookieInfo.OriginalCookie) {
- CommittingCookies[cookieInfo.OriginalCookie] = cookie;
- originalCookies.push_back(cookieInfo.OriginalCookie);
- cookieInfo.OriginalCookie = 0;
- for (auto* lock : cookieInfo.Locks) {
- lock->Cookies.erase(cookie);
- }
- } else {
- commitedCookies.push_back(cookie);
- }
- } else if (cookie >= CookieCounter) {
- Destroy("Unknown cookie", NErrorCode::BAD_REQUEST);
- break;
- } else {
- commitedCookies.push_back(cookie);
- }
- }
- if (!originalCookies.empty()) {
- Consumer->Commit(originalCookies);
- }
- if (!commitedCookies.empty()) {
- FastCommit(commitedCookies);
- }
- DEBUG_LOG(FormatCommitForLog("", cookies, originalCookies, commitedCookies), "", SessionId);
- // clean commited cookies
- while (!Cookies.empty() && 0 == Cookies.front().OriginalCookie) {
- Cookies.pop_front();
- }
-}
-
-void TRetryingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- if (!Consumer) {
- // client should receive message about release lock
- return;
- }
- auto lockIt = Locks.find(std::make_pair(topic, partition));
- if (lockIt == Locks.end() || !lockIt->second.Locked) {
- WARN_LOG("Requesting partition status on partition without lock. Topic: " << topic << ". Partition: " << partition, "", SessionId);
- } else if (lockIt->second.Gen != generation) {
- WARN_LOG("Requesting partition status on partition with wrong generation of lock. Topic: " << topic << ". Partition: " << partition << ". Generation: " << generation, "", SessionId);
- } else {
- Consumer->RequestPartitionStatus(topic, partition, lockIt->second.OriginalGen);
- }
-}
-
-void TRetryingConsumer::Cancel() {
- Destroy(GetCancelReason());
-}
-
-NThreading::TFuture<void> TRetryingConsumer::Destroyed() noexcept {
- return ConsumerDestroyedPromise.GetFuture();
-}
-
-void TRetryingConsumer::OnStartDeadline() {
- if (!StartPromise.HasValue()) {
- TError error;
- error.SetDescription("Start timeout.");
- error.SetCode(NErrorCode::CREATE_TIMEOUT);
- Destroy(error);
- }
-}
-
-void TRetryingConsumer::OnConsumerDead(const TError& error) {
- WARN_LOG("Subconsumer is dead: " << error, "", SessionId);
- ScheduleReconnect();
-}
-
-void TRetryingConsumer::ScheduleReconnect() {
- if (Stopping || Reconnecting) {
- return;
- }
- Reconnecting = true;
- if (!Locks.empty()) {
- // need to notify client that all locks are expired
- for (auto&& [key, lock] : Locks) {
- if (lock.Locked) {
- TReadResponse response;
- auto* release = response.MutableRelease();
- release->set_generation(lock.Gen);
- release->set_topic(key.first);
- release->set_partition(key.second);
- release->set_can_commit(false);
- FastResponse(TConsumerMessage(std::move(response)));
- }
- }
- }
- if (!CommittingCookies.empty()) {
- // need to notify client that all cookies are commited, because these cookies aren't valid anymore
- TReadResponse response;
- auto* cookies = response.MutableCommit()->MutableCookie();
- cookies->Reserve(CommittingCookies.size());
- for (const auto& cookiePair : CommittingCookies) {
- cookies->Add(cookiePair.second);
- }
- FastResponse(TConsumerMessage(std::move(response)));
- }
-
- Cookies.clear();
- CommittingCookies.clear();
- Locks.clear();
- Consumer = nullptr;
-
- if (ReconnectionAttemptsDone >= Settings.MaxAttempts) {
- Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts");
- return;
- }
-
- ++ReconnectionAttemptsDone;
-
- TDuration delay = Min(Settings.MaxReconnectionDelay, ReconnectionAttemptsDone * Settings.ReconnectionDelay);
- std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this();
- PQLib->GetScheduler().Schedule(delay, this, [weakRef]() {
- auto self = weakRef.lock();
- if (nullptr != self) {
- self->DoReconnect(self->Settings.StartSessionTimeout.ToDeadLine());
- self->Reconnecting = false;
- }
- });
-}
-
-void TRetryingConsumer::DoReconnect(TInstant deadline) {
- if (Stopping) {
- return;
- }
- DEBUG_LOG("Create subconsumer", "", SessionId);
- Consumer = PQLib->CreateRawConsumer(Settings, DestroyEventRef, Logger);
- std::weak_ptr<TRetryingConsumer> weak(shared_from_this());
- StartFuture = Consumer->Start(deadline);
- PQLib->Subscribe(StartFuture, this, [weak](const NThreading::TFuture<TConsumerCreateResponse>& f) {
- auto self = weak.lock();
- if (nullptr != self) {
- self->StartProcessing(f);
- }
- });
-
- PQLib->Subscribe(Consumer->IsDead(), this, [weak](const NThreading::TFuture<TError>& error) {
- auto self = weak.lock();
- if (nullptr != self) {
- self->OnConsumerDead(error.GetValue());
- }
- });
-}
-
-void TRetryingConsumer::StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f) {
- Y_VERIFY(f.HasValue());
- if (f.GetValue().Response.HasError()) {
- WARN_LOG("Cannot create subconsumer: " << f.GetValue().Response.GetError(), "", SessionId);
- if (IsRetryable(f.GetValue().Response.GetError())) {
- ScheduleReconnect();
- } else {
- Destroy(f.GetValue().Response.GetError());
- }
- } else {
- if (!SessionId) {
- SessionId = f.GetValue().Response.GetInit().GetSessionId();
- }
- ReconnectionAttemptsDone = 0;
- // need to schedule again all pending requests which are running before connection lost
- for (size_t cnt = PendingRequests.size(); 0 != cnt; --cnt) {
- DoRequest();
- }
- StartPromise.TrySetValue(f.GetValue());
- }
-}
-
-void TRetryingConsumer::SubscribeDestroyed() {
- NThreading::TPromise<void> promise = ConsumerDestroyedPromise;
- auto handler = [promise](const auto&) mutable {
- promise.SetValue();
- };
- if (Consumer) {
- WaitExceptionOrAll(DestroyedPromise.GetFuture(), Consumer->Destroyed()).Subscribe(handler);
- } else {
- DestroyedPromise.GetFuture().Subscribe(handler);
- }
-
- DestroyPQLibRef();
-}
-
-void TRetryingConsumer::Destroy(const TError& error) {
- if (Stopping) {
- return;
- }
-
- Stopping = true;
-
- SubscribeDestroyed();
-
- TConsumerMessage message = [&]() {
- TReadResponse response;
- *response.MutableError() = error;
- return TConsumerMessage(std::move(response));
- }();
-
- for (auto& r : PendingRequests) {
- r.TrySetValue(message);
- }
- IsDeadPromise.TrySetValue(error);
- if (StartPromise.Initialized()) {
- StartPromise.TrySetValue(TConsumerCreateResponse(std::move(message.Response)));
- }
- PendingRequests.clear();
- ReadyResponses.clear();
- Cookies.clear();
- CommittingCookies.clear();
- Locks.clear();
- Consumer = nullptr;
- Reconnecting = false;
-}
-
-void TRetryingConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) {
- TError error;
- error.SetDescription(description);
- error.SetCode(code);
- Destroy(error);
-}
-
-void TRetryingConsumer::DoRequest() {
- Y_VERIFY(Consumer);
- std::weak_ptr<TRetryingConsumer> weak(shared_from_this());
- PQLib->Subscribe(Consumer->GetNextMessage(), this, [weak](NThreading::TFuture<TConsumerMessage>& f) mutable {
- auto self = weak.lock();
- if (nullptr != self) {
- self->ProcessResponse(f.ExtractValueSync());
- }
- });
-}
-
-void TRetryingConsumer::ProcessResponse(TConsumerMessage&& message) {
- switch (message.Type) {
- case NPersQueue::EMessageType::EMT_DATA:
- {
- auto* data = message.Response.mutable_data();
- TCookieInfo* cookie = nullptr;
- if (!Settings.CommitsDisabled) {
- Cookies.push_back(TCookieInfo{{}, data->cookie(), CookieCounter++});
- cookie = &Cookies.back();
- data->set_cookie(cookie->UserCookie);
- DEBUG_LOG("Got data from subconsumer. Cookie: " << cookie->OriginalCookie << ". User cookie: " << cookie->UserCookie, "", SessionId);
- }
-
- for (auto& b : data->message_batch()) {
- auto lockIt = Locks.find(std::make_pair(b.topic(), b.partition()));
- if (lockIt != Locks.end()) {
- Y_VERIFY(lockIt->second.Locked);
- if (!Settings.CommitsDisabled) {
- Y_ASSERT(nullptr != cookie);
- cookie->Locks.emplace(&lockIt->second);
- lockIt->second.Cookies.emplace(cookie->UserCookie);
- }
-
- // Validate that all offsets are >= LockInfo.ReadOffset as expected.
- if (lockIt->second.ReadOffset != 0) {
- for (const auto& message : b.message()) {
- if (message.offset() < lockIt->second.ReadOffset) {
- Destroy(
- TStringBuilder()
- << "Fatal error: expected offsets for topic " << b.topic()
- << " partition " << b.partition() << " >= " << lockIt->second.ReadOffset
- << ", but got offset " << message.offset()
- );
- return;
- }
- }
- }
- }
- }
- // TODO (bulatman) what about batched_data?
- }
- break;
- case NPersQueue::EMessageType::EMT_LOCK:
- {
- auto* lock = message.Response.mutable_lock();
- auto& lockInfo = Locks[std::make_pair(lock->topic(), lock->partition())];
- Y_VERIFY(!lockInfo.Locked);
- lockInfo.Locked = true;
- lockInfo.Gen = GenCounter++;
- lockInfo.OriginalGen = lock->generation();
-
- lock->set_generation(lockInfo.Gen);
-
- DEBUG_LOG("Got lock from subconsumer on (" << lock->topic() << ", " << lock->partition() << "). Generation: " << lockInfo.OriginalGen << ". User generation: " << lockInfo.Gen, "", SessionId);
-
- std::weak_ptr<TRetryingConsumer> weak = shared_from_this();
- PQLib->Subscribe(message.ReadyToRead.GetFuture(), this,
- [weak, topic = lock->topic(), partition = lock->partition(), generation = lock->generation()](const NThreading::TFuture<NPersQueue::TLockInfo>& lockInfo) {
- auto self = weak.lock();
- if (nullptr != self) {
- self->UpdateReadyToRead(lockInfo.GetValue(), topic, partition, generation);
- }
- });
- }
- break;
- case NPersQueue::EMessageType::EMT_RELEASE:
- {
- auto* release = message.Response.mutable_release();
- const bool softRelease = release->can_commit();
- auto lockIt = Locks.find(std::make_pair(release->topic(), release->partition()));
- const bool lockInfoFound = lockIt != Locks.end();
- DEBUG_LOG("Got release from subconsumer on (" << release->topic() << ", " << release->partition() << "). Can commit: " << softRelease << ". Has lock info: " << lockInfoFound << ". Generation: " << release->generation() << ". User generation: " << (lockInfoFound ? lockIt->second.Gen : 0), "", SessionId);
- if (lockInfoFound) { // It is normal situation when client receives Release(canCommit=true) and then Release(canCommit=false).
- auto& lockInfo = lockIt->second;
- Y_VERIFY(lockInfo.OriginalGen == release->generation(), "lock generation mismatch");
- release->set_generation(lockInfo.Gen);
-
- if (softRelease) {
- lockInfo.Locked = false;
- } else {
- for (auto cookie : lockInfo.Cookies) {
- auto& cookieInfo = Cookies[cookie + Cookies.size() - CookieCounter];
- Y_VERIFY(cookieInfo.UserCookie == cookie);
- // Lock is not valid anymore.
- cookieInfo.Locks.erase(&lockInfo);
- }
- Locks.erase(lockIt);
- }
- } else {
- return;
- }
- }
- break;
- case NPersQueue::EMessageType::EMT_STATUS:
- {
- auto* status = message.Response.mutable_partition_status();
- auto lockIt = Locks.find(std::make_pair(status->topic(), status->partition()));
- Y_VERIFY(lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.OriginalGen == status->generation());
- status->set_generation(lockIt->second.Gen);
- }
- break;
- case NPersQueue::EMessageType::EMT_COMMIT:
- {
- auto* cookies = message.Response.mutable_commit()->mutable_cookie();
- // convert cookies
- for (int i = 0; i < cookies->size(); ++i) {
- auto it = CommittingCookies.find(cookies->Get(i));
- Y_VERIFY(it != CommittingCookies.end(), "unknown commited cookie!");
- cookies->Set(i, it->second);
- CommittingCookies.erase(it);
- }
- }
- break;
-
- case NPersQueue::EMessageType::EMT_ERROR:
- // check error, if retryable, need to recreate consumer and read message again
- const bool retryable = IsRetryable(message.Response.error());
- DEBUG_LOG("Got error from subconsumer: " << message.Response.error() << ". Retryable: " << retryable, "", SessionId);
- if (retryable) {
- ScheduleReconnect();
- return;
- }
- break;
- };
-
- if (!PendingRequests.empty()) {
- PendingRequests.front().SetValue(std::move(message));
- PendingRequests.pop_front();
- } else {
- ReadyResponses.push_back(std::move(message));
- }
-}
-
-void TRetryingConsumer::FastResponse(TConsumerMessage&& message) {
- if (!PendingRequests.empty()) {
- PendingRequests.front().SetValue(std::move(message));
- PendingRequests.pop_front();
- } else {
- ReadyResponses.push_back(std::move(message));
- }
-}
-
-void TRetryingConsumer::FastCommit(const TVector<ui64>& cookies) {
- TReadResponse response;
- auto* cookiesMessage = response.MutableCommit()->MutableCookie();
- cookiesMessage->Reserve(cookies.size());
- for (auto cookie : cookies) {
- cookiesMessage->Add(cookie);
- }
- FastResponse(TConsumerMessage(std::move(response)));
-}
-
-void TRetryingConsumer::UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation) {
- const auto lockIt = Locks.find(std::make_pair(topic, partition));
- if (lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.Gen == generation) {
- lockIt->second.ReadOffset = readyToRead.ReadOffset;
- }
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h
deleted file mode 100644
index 448d35e4c8..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h
+++ /dev/null
@@ -1,107 +0,0 @@
-#pragma once
-
-#include "consumer.h"
-#include "scheduler.h"
-#include "internals.h"
-#include "persqueue_p.h"
-#include "iconsumer_p.h"
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/vector.h>
-
-#include <deque>
-
-namespace NPersQueue {
-
-// @brief consumer implementation which transparently retries all connectivity errors
-class TRetryingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TRetryingConsumer> {
- // @brief locked partitions info
- struct TLockInfo {
- THashSet<ui64> Cookies; // related cookies
- ui64 Gen = 0;
- ui64 OriginalGen = 0; // original generation
- ui64 ReadOffset = 0; // read offset specified by client
- bool Locked = false;
- };
-
- struct TCookieInfo {
- THashSet<TLockInfo*> Locks; // related locks
- ui64 OriginalCookie = 0; // zero means invalid cookie
- ui64 UserCookie = 0;
- };
-
-public:
- TRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef,
- TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger);
-
- ~TRetryingConsumer() noexcept override;
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override;
-
- using IConsumerImpl::GetNextMessage;
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override;
-
- void Commit(const TVector<ui64>& cookies) noexcept override;
-
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override;
-
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- void Cancel() override;
-
- NThreading::TFuture<void> Destroyed() noexcept override;
-
-private:
- void OnStartDeadline();
- void OnConsumerDead(const TError& error);
- void ScheduleReconnect();
- void DoReconnect(TInstant deadline);
- void StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f);
- void SubscribeDestroyed();
- void Destroy(const TError& error);
- void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR);
- void DoRequest();
- void ProcessResponse(TConsumerMessage&& message);
- void FastResponse(TConsumerMessage&& message);
- void FastCommit(const TVector<ui64>& cookies);
- void UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation);
-
-private:
- TConsumerSettings Settings;
- TIntrusivePtr<ILogger> Logger;
- std::shared_ptr<IConsumerImpl> Consumer;
- TString SessionId;
-
- NThreading::TFuture<TConsumerCreateResponse> StartFuture;
- NThreading::TPromise<TConsumerCreateResponse> StartPromise;
- NThreading::TPromise<TError> IsDeadPromise;
- NThreading::TPromise<void> ConsumerDestroyedPromise;
-
- // requests which are waiting for response
- std::deque<NThreading::TPromise<TConsumerMessage>> PendingRequests;
- // ready messages for returning to clients immediately
- std::deque<TConsumerMessage> ReadyResponses;
- // active cookies
- std::deque<TCookieInfo> Cookies;
- THashMap<ui64, ui64> CommittingCookies;
- // active data per topic, partition
- THashMap<std::pair<TString, ui64>, TLockInfo> Locks; // topic, partition -> LockInfo
-
- // number of unsuccessful retries after last error
- ui64 GenCounter;
- ui64 CookieCounter;
- unsigned ReconnectionAttemptsDone;
- // destroying process started
- bool Stopping;
- // reconnecting in process
- bool Reconnecting;
-};
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp
deleted file mode 100644
index 8065ed717a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp
+++ /dev/null
@@ -1,604 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/guid.h>
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
-namespace NPersQueue {
- Y_UNIT_TEST_SUITE(TRetryingConsumerTest) {
- static TConsumerSettings FakeSettings() {
- TConsumerSettings settings;
- settings.ReconnectOnFailure = true;
- settings.Server = TServerSetting{"localhost"};
- settings.Topics.push_back("topic");
- settings.ClientId = "client";
- return settings;
- }
-
- Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) {
- // Test that consumer doesn't hang on till shutdown
- TPQLib lib;
- auto settings = FakeSettings();
- lib.CreateConsumer(settings);
- }
-
- Y_UNIT_TEST(StartDeadlineExpires) {
- TPQLibSettings libSettings;
- libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1);
- libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG);
- TPQLib lib(libSettings);
- THolder<IConsumer> consumer;
- {
- auto settings = FakeSettings();
- settings.ReconnectionDelay = TDuration::MilliSeconds(1);
- settings.StartSessionTimeout = TDuration::MilliSeconds(10);
- if (GrpcV1EnabledByDefault()) {
- settings.MaxAttempts = 3;
- }
- consumer = lib.CreateConsumer(settings);
- }
- const TInstant beforeStart = TInstant::Now();
- auto future = consumer->Start(TDuration::MilliSeconds(100));
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT(future.GetValueSync().Response.HasError());
- UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT,
- "Error: " << future.GetValueSync().Response.GetError());
- const TInstant now = TInstant::Now();
- UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now);
- }
- auto isDead = consumer->IsDead();
- isDead.Wait();
-
- DestroyAndWait(consumer);
- }
-
- Y_UNIT_TEST(StartMaxAttemptsExpire) {
- TPQLibSettings libSettings;
- libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1);
- libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG);
- TPQLib lib(libSettings);
- THolder<IConsumer> consumer;
- {
- auto settings = FakeSettings();
- settings.ReconnectionDelay = TDuration::MilliSeconds(10);
- settings.StartSessionTimeout = TDuration::MilliSeconds(10);
- settings.MaxAttempts = 3;
- consumer = lib.CreateConsumer(settings);
- }
- const TInstant beforeStart = TInstant::Now();
- auto future = consumer->Start();
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT(future.GetValueSync().Response.HasError());
- }
- auto deadFuture = consumer->IsDead();
- deadFuture.Wait();
- const TInstant now = TInstant::Now();
- UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now);
-
- DestroyAndWait(consumer);
- }
-
- static void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response);
- }
-
- static void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response);
- }
-
- static void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response);
- }
-
- static void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response);
- }
-
- static TProducerSettings MakeProducerSettings(const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = false;
- producerSettings.Topic = "topic1";
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::LZOP;
- return producerSettings;
- }
-
- static TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) {
- TConsumerSettings settings;
- settings.ReconnectOnFailure = true;
- settings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- settings.Topics.emplace_back("topic1");
- settings.ClientId = "user";
- return settings;
- }
-
- Y_UNIT_TEST(ReconnectsToServer) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- const size_t partitions = 10;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
-
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib PQLib;
-
- auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false);
- UNIT_ASSERT(!producer->IsDead().HasValue());
- producer->Start().Wait();
-
- auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false);
- TFuture<TError> isDead = consumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
- consumer->Start().Wait();
-
- auto read1 = consumer->GetNextMessage();
-
- // write first value
- auto write1 = producer->Write(1, TString("blob1"));
- AssertWriteValid(write1);
- AssertReadValid(read1);
-
- // commit
- consumer->Commit({1});
- auto commitAck = consumer->GetNextMessage();
- Cerr << "wait for commit1\n";
-
- AssertCommited(commitAck);
-
- Cerr << "After shutdown\n";
- testServer.ShutdownGRpc();
-
- auto read2 = consumer->GetNextMessage();
- UNIT_ASSERT(!isDead.HasValue());
- testServer.EnableGRpc();
-
- testServer.WaitInit("topic1");
-
- Cerr << "Wait producer1 death\n";
-
- producer->IsDead().Wait();
-
- producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false);
- Cerr << "Wait producer2 start\n";
- producer->Start().Wait();
- UNIT_ASSERT(!producer->IsDead().HasValue());
- auto write2 = producer->Write(2, TString("blob2"));
- Cerr << "Wait for second write\n";
- AssertWriteValid(write2);
- Cerr << "wait for read second blob\n";
- AssertReadValid(read2);
- UNIT_ASSERT_VALUES_EQUAL(2u, read2.GetValueSync().Response.GetData().GetCookie());
- UNIT_ASSERT(!isDead.HasValue());
- consumer->Commit({2});
- Cerr << "wait for commit2\n";
- auto commitAck2 = consumer->GetNextMessage();
- AssertCommited(commitAck2);
-
- DestroyAndWait(producer);
- DestroyAndWait(consumer);
- }
-
- static void DiesOnTooManyReconnectionAttempts(bool callRead) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2);
-
- testServer.WaitInit("topic1");
-
- TConsumerSettings settings = MakeConsumerSettings(testServer);
- settings.MaxAttempts = 3;
- settings.ReconnectionDelay = TDuration::MilliSeconds(100);
- auto consumer = testServer.PQLib->CreateConsumer(settings, testServer.PQLibSettings.DefaultLogger, false);
- consumer->Start().Wait();
- TFuture<TError> isDead = consumer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- // shutdown server
- const TInstant beforeShutdown = TInstant::Now();
- testServer.ShutdownServer();
-
- NThreading::TFuture<TConsumerMessage> read;
- if (callRead) {
- read = consumer->GetNextMessage();
- }
-
- isDead.Wait();
- const TInstant afterDead = TInstant::Now();
- // 3 attempts: 100ms, 200ms and 300ms
- UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(GrpcV1EnabledByDefault() ? 300 : 600), "real difference: " << (afterDead - beforeShutdown));
-
- if (callRead) {
- AssertReadFailed(read);
- }
- }
-
- Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutRead) {
- // Check that we reconnect even without explicit write errors
- DiesOnTooManyReconnectionAttempts(false);
- }
-
- Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithRead) {
- DiesOnTooManyReconnectionAttempts(true);
- }
-
- Y_UNIT_TEST(ReadBeforeConnectToServer) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- const size_t partitions = 10;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
-
- testServer.WaitInit("topic1");
-
- auto consumer = testServer.PQLib->CreateConsumer(
- MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false
- );
- auto consumerStarted = consumer->Start();
- auto read1 = consumer->GetNextMessage();
- consumerStarted.Wait();
- TVector<TFuture<TConsumerMessage>> messages;
- for (int i = 2; i <= 5; ++i) {
- messages.push_back(consumer->GetNextMessage());
- }
-
- auto producer = testServer.PQLib->CreateProducer(
- MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false
- );
- producer->Start().Wait();
-
- while (!messages.back().HasValue()) {
- auto write = producer->Write(TString("data"));
- AssertWriteValid(write);
- }
-
- AssertReadValid(read1);
- for (auto& msg: messages) {
- AssertReadValid(msg);
- }
-
- DestroyAndWait(producer);
- DestroyAndWait(consumer);
- }
-
- Y_UNIT_TEST(CancelsStartAfterPQLibDeath) {
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto consumer = PQLib->CreateConsumer(FakeSettings(), logger, false);
- auto start = consumer->Start();
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT(!start.HasValue());
- }
-
- PQLib = nullptr;
-
- if (!GrpcV1EnabledByDefault()) {
- UNIT_ASSERT(start.HasValue());
- UNIT_ASSERT(start.GetValue().Response.HasError());
- }
-
- auto dead = consumer->IsDead();
- if (GrpcV1EnabledByDefault()) {
- dead.Wait();
- } else {
- UNIT_ASSERT(dead.HasValue());
- }
-
- auto read = consumer->GetNextMessage();
- UNIT_ASSERT(read.HasValue());
- UNIT_ASSERT(read.GetValueSync().Response.HasError());
- }
-
- Y_UNIT_TEST(LockSession) {
- SDKTestSetup setup("LockSession", false);
- setup.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::MilliSeconds(100));
- setup.Start();
- auto consumerSettings = setup.GetConsumerSettings();
- consumerSettings.UseLockSession = true;
- consumerSettings.ReconnectOnFailure = true;
- auto consumer = setup.StartConsumer(consumerSettings);
-
- setup.WriteToTopic({"msg1", "msg2"});
-
- size_t releases = 0;
- for (size_t times = 0; times < 2; ++times) {
- while (true) {
- auto msgFuture = consumer->GetNextMessage();
- auto& msg = msgFuture.GetValueSync();
- setup.GetLog() << TLOG_INFO << "Got response from consumer: " << msg.Response;
- if (times) {
- UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA || msg.Type == EMT_RELEASE, msg.Response);
- } else {
- UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA, msg.Response);
- }
- if (msg.Type == EMT_LOCK) {
- TLockInfo lockInfo;
- lockInfo.ReadOffset = 1;
- msg.ReadyToRead.SetValue(lockInfo);
- } else if (msg.Type == EMT_DATA) {
- UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().MessageBatchSize(), 1, msg.Response);
- UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).MessageSize(), 1, msg.Response);
- UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg2", msg.Response);
- break;
- } else if (msg.Type == EMT_RELEASE) {
- ++releases;
- }
- }
- if (!times) { // force reconnect
- setup.ShutdownGRpc();
- setup.EnableGRpc();
- }
- }
- UNIT_ASSERT_VALUES_EQUAL(releases, 1);
- }
-
- Y_UNIT_TEST(ReadAhead) {
- SDKTestSetup setup("ReadAhead");
-
- auto consumerSettings = setup.GetConsumerSettings();
- consumerSettings.ReconnectOnFailure = true;
- auto consumer = setup.StartConsumer(consumerSettings);
-
- setup.WriteToTopic({"msg"});
-
- // Wait until read ahead occurs.
- Sleep(TDuration::MilliSeconds(100));
-
- setup.ReadFromTopic({{"msg"}}, true, consumer.Get());
- }
-
- Y_UNIT_TEST(ReadBeforeWrite) {
- SDKTestSetup setup("ReadBeforeWrite");
-
- auto consumerSettings = setup.GetConsumerSettings();
- consumerSettings.ReconnectOnFailure = true;
- auto consumer = setup.StartConsumer(consumerSettings);
-
- auto msgFuture = consumer->GetNextMessage();
-
- Sleep(TDuration::MilliSeconds(100));
-
- setup.WriteToTopic({"msg"});
-
- const auto& msg = msgFuture.GetValueSync();
- UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA);
- UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg");
- }
-
- Y_UNIT_TEST(CommitDisabled) {
- SDKTestSetup setup("CommitDisabled");
-
- auto consumerSettings = setup.GetConsumerSettings();
- consumerSettings.ReconnectOnFailure = true;
- consumerSettings.CommitsDisabled = true;
- auto consumer = setup.StartConsumer(consumerSettings);
- setup.WriteToTopic({"msg", "msg2"});
- setup.ReadFromTopic({{"msg", "msg2"}}, false, consumer.Get());
- // check that commit fails with error when commits disabled
- setup.WriteToTopic({"msg3"});
- auto msgFuture = consumer->GetNextMessage();
- const auto& msg = msgFuture.GetValueSync();
- UNIT_ASSERT_C(!msg.Response.HasError(), msg.Response);
- UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA);
- UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg3");
- consumer->Commit({msg.Response.GetData().GetCookie()});
- auto isDead = consumer->IsDead();
- auto& err = isDead.GetValueSync();
- UNIT_ASSERT_STRINGS_EQUAL("Commits are disabled", err.GetDescription());
- }
-
- void LockReleaseTest(bool multiclusterConsumer, bool killPqrb) {
- SDKTestSetup setup("LockRelease");
- const TString topicName = "lock_release";
- const size_t partitionsCount = 10;
- setup.CreateTopic(topicName, setup.GetLocalCluster(), partitionsCount);
-
- auto consumerSettings = setup.GetConsumerSettings();
- consumerSettings.Topics[0] = topicName;
- consumerSettings.ReconnectOnFailure = true;
- consumerSettings.UseLockSession = true;
- if (multiclusterConsumer) { // The same test for multicluster wrapper.
- consumerSettings.ReadFromAllClusterSources = true;
- consumerSettings.ReadMirroredPartitions = false;
- }
- auto consumer = setup.StartConsumer(consumerSettings);
-
- std::vector<int> partitionsLocked(partitionsCount * 2); // For both consumers.
- std::vector<ui64> partitionsGens(partitionsCount * 2);
- for (size_t i = 0; i < partitionsCount; ++i) {
- auto lock = consumer->GetNextMessage().GetValueSync();
- UNIT_ASSERT_EQUAL_C(lock.Type, EMT_LOCK, lock.Response);
- UNIT_ASSERT_C(!partitionsLocked[lock.Response.GetLock().GetPartition()], lock.Response);
- partitionsLocked[lock.Response.GetLock().GetPartition()] = 1;
- partitionsGens[lock.Response.GetLock().GetPartition()] = lock.Response.GetLock().GetGeneration();
- lock.ReadyToRead.SetValue({});
- }
-
- const size_t messagesCount = 100;
- auto generateMessages = [&]() {
- for (size_t i = 0; i < messagesCount; ++i) { // Write to several random partitions.
- const TString uniqStr = CreateGuidAsString();
- auto settings = setup.GetProducerSettings();
- settings.Topic = topicName;
- settings.SourceId = uniqStr;
- auto producer = setup.StartProducer(settings);
- setup.WriteToTopic({uniqStr}, producer.Get());
- }
- };
-
- generateMessages();
-
- THolder<IConsumer> otherConsumer = nullptr;
- bool rebalanced = false;
-
- NThreading::TFuture<TConsumerMessage> msg1 = consumer->GetNextMessage();
- NThreading::TFuture<TConsumerMessage> msg2;
- size_t hardReleasesCount = 0;
-
- THashSet<TString> msgsReceived;
- THashSet<ui64> cookiesToCommit;
- auto locksCount = [&]() -> int {
- return std::accumulate(partitionsLocked.begin(), partitionsLocked.end(), 0);
- };
- auto allEventsReceived = [&]() -> bool {
- return msgsReceived.size() >= messagesCount * 2
- && locksCount() == partitionsCount
- && (!killPqrb || hardReleasesCount > 0)
- && cookiesToCommit.empty();
- };
-
- int iterationNum = 0;
- while (!allEventsReceived()) {
- ++iterationNum;
- Cerr << "\nIterationNum: " << iterationNum << Endl;
- Cerr << "msgsReceived.size(): " << msgsReceived.size() << Endl;
- Cerr << "partitionsCount: " << partitionsCount << Endl;
- Cerr << "locksGot: " << locksCount() << Endl;
- Cerr << "hardReleasesCount: " << hardReleasesCount << Endl;
- Cerr << "cookiesToCommit.size(): " << cookiesToCommit.size() << Endl;
- auto waiter = msg2.Initialized() ? NThreading::WaitAny(msg1.IgnoreResult(), msg2.IgnoreResult()) : msg1.IgnoreResult();
- waiter.Wait();
- auto processValue = [&](auto& msg, auto& consumer) {
- const bool isOtherConsumer = consumer == otherConsumer;
- const ui64 consumerMix = isOtherConsumer ? 0x8000000000000000 : 0;
-
- switch (msg.GetValue().Type) {
- case EMT_LOCK:
- {
- UNIT_ASSERT_LT(msg.GetValue().Response.GetLock().GetPartition(), partitionsCount);
- const size_t idx = msg.GetValue().Response.GetLock().GetPartition() + (isOtherConsumer ? partitionsCount : 0);
- if (msg.GetValue().Response.GetLock().GetGeneration() >= partitionsGens[idx]) {
- UNIT_ASSERT_C(!partitionsLocked[idx], msg.GetValue().Response << ". Other: " << isOtherConsumer);
- partitionsLocked[idx] = 1;
- msg.GetValue().ReadyToRead.SetValue({});
- partitionsGens[idx] = msg.GetValue().Response.GetLock().GetGeneration();
- }
- break;
- }
- case EMT_RELEASE:
- {
- UNIT_ASSERT_LT(msg.GetValue().Response.GetRelease().GetPartition(), partitionsCount);
- const size_t idx = msg.GetValue().Response.GetRelease().GetPartition() + (isOtherConsumer ? partitionsCount : 0);
- partitionsLocked[idx] = 0;
- if (!msg.GetValue().Response.GetRelease().GetCanCommit()) {
- ++hardReleasesCount;
- }
- if (!killPqrb) {
- UNIT_ASSERT(msg.GetValue().Response.GetRelease().GetCanCommit()); // No restarts => soft release.
- }
- break;
- }
- case EMT_DATA:
- {
- const auto& resp = msg.GetValue().Response.GetData();
- UNIT_ASSERT(cookiesToCommit.insert(resp.GetCookie() | consumerMix).second);
- for (const auto& batch : resp.GetMessageBatch()) {
- for (const auto& message : batch.GetMessage()) {
- msgsReceived.insert(message.GetData());
- }
- }
- consumer->Commit({resp.GetCookie()});
- break;
- }
- case EMT_ERROR:
- case EMT_STATUS:
- case EMT_COMMIT:
- for (ui64 cookie : msg.GetValue().Response.GetCommit().GetCookie()) {
- UNIT_ASSERT(cookiesToCommit.find(cookie | consumerMix) != cookiesToCommit.end());
- cookiesToCommit.erase(cookie | consumerMix);
- }
- break;
- }
- msg = consumer->GetNextMessage();
- };
-
- if (msg1.HasValue()) {
- processValue(msg1, consumer);
- }
- if (msg2.Initialized() && msg2.HasValue()) {
- processValue(msg2, otherConsumer);
- }
-
- if (!otherConsumer && msgsReceived.size() >= messagesCount / 4) {
- otherConsumer = setup.StartConsumer(consumerSettings);
- msg2 = otherConsumer->GetNextMessage();
- }
-
- if (!rebalanced && msgsReceived.size() >= messagesCount / 2) {
- rebalanced = true;
-
- if (killPqrb) {
- Sleep(TDuration::MilliSeconds(100));
- setup.KillPqrb(topicName, setup.GetLocalCluster());
- }
-
- generateMessages();
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL(locksCount(), partitionsCount);
- if (killPqrb) {
- UNIT_ASSERT(hardReleasesCount > 0);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(hardReleasesCount, 0);
- }
- }
-
- Y_UNIT_TEST(LockRelease) {
- LockReleaseTest(false, false);
- }
-
- Y_UNIT_TEST(LockReleaseHardRelease) {
- LockReleaseTest(false, true);
- }
-
- Y_UNIT_TEST(LockReleaseMulticluster) {
- LockReleaseTest(true, false);
- }
-
- Y_UNIT_TEST(LockReleaseMulticlusterHardRelease) {
- LockReleaseTest(true, true);
- }
-
- Y_UNIT_TEST(NonRetryableErrorOnStart) {
- SDKTestSetup setup("NonRetryableErrorOnStart");
-
- auto consumerSettings = setup.GetConsumerSettings();
- //consumerSettings.
- consumerSettings.ReconnectOnFailure = true;
- consumerSettings.Topics = { "unknown/topic" };
- THolder<IConsumer> consumer = setup.GetPQLib()->CreateConsumer(consumerSettings);
- auto startFuture = consumer->Start();
- if (!GrpcV1EnabledByDefault()) {
- Cerr << "===Got response: " << startFuture.GetValueSync().Response << Endl;
- UNIT_ASSERT_C(startFuture.GetValueSync().Response.HasError(), startFuture.GetValueSync().Response);
- UNIT_ASSERT_EQUAL(startFuture.GetValueSync().Response.GetError().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC);
- }
- auto deadFuture = consumer->IsDead();
- UNIT_ASSERT_EQUAL_C(deadFuture.GetValueSync().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC, deadFuture.GetValueSync());
- }
- }
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp
deleted file mode 100644
index bb562ea0ae..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp
+++ /dev/null
@@ -1,370 +0,0 @@
-#include "retrying_producer.h"
-#include "persqueue_p.h"
-
-#include <util/generic/strbuf.h>
-#include <util/stream/zlib.h>
-#include <util/string/cast.h>
-#include <util/string/printf.h>
-#include <util/string/vector.h>
-#include <util/string/builder.h>
-
-namespace NPersQueue {
-
-TRetryingProducer::TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef,
- TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger)
- : IProducerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , Logger(std::move(logger))
- , IsDeadPromise(NThreading::NewPromise<TError>())
- , NeedRecreation(true)
- , Stopping(false)
- , ToProcess(0)
- , LastReconnectionDelay(TDuration::Zero())
- , ReconnectionAttemptsDone(0)
-{
- if (Settings.MaxAttempts == 0) {
- ythrow yexception() << "MaxAttempts setting can't be zero.";
- }
-
- if (Settings.ReconnectionDelay == TDuration::Zero()) {
- ythrow yexception() << "ReconnectionDelay setting can't be zero.";
- }
-
- if (Settings.StartSessionTimeout < PQLib->GetSettings().ChannelCreationTimeout) {
- ythrow yexception() << "StartSessionTimeout can't be less than ChannelCreationTimeout.";
- }
-}
-
-TRetryingProducer::~TRetryingProducer() noexcept {
- Destroy("Destructor called");
-}
-
-NThreading::TFuture<TProducerCreateResponse> TRetryingProducer::Start(TInstant deadline) noexcept {
- StartPromise = NThreading::NewPromise<TProducerCreateResponse>();
- if (deadline != TInstant::Max()) {
- std::weak_ptr<TRetryingProducer> self = shared_from_this();
- auto onDeadline = [self] {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnStartDeadline();
- }
- };
- PQLib->GetScheduler().Schedule(deadline, this, onDeadline);
- }
- RecreateProducer(deadline);
- return StartPromise.GetFuture();
-}
-
-void TRetryingProducer::RecreateProducer(TInstant deadline) noexcept {
- Producer = nullptr;
- if (Stopping)
- return;
-
- Y_VERIFY(InFlightRequests.size() == Futures.size());
- DEBUG_LOG("Recreating subproducer. Futures size: " << Futures.size(), Settings.SourceId, "");
- if (Futures.empty()) {
- DoRecreate(deadline);
- } // otherwise it will be recreated in ProcessFutures()
-}
-
-void TRetryingProducer::DoRecreate(TInstant deadline) noexcept {
- Y_VERIFY(InFlightRequests.size() == Futures.size());
- Y_VERIFY(Futures.empty());
- if (Stopping)
- return;
- Y_VERIFY(NeedRecreation);
- NeedRecreation = false;
- if (ReconnectionAttemptsDone >= Settings.MaxAttempts) {
- Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts");
- return;
- }
- ++ReconnectionAttemptsDone;
- DEBUG_LOG("Creating subproducer. Attempt: " << ReconnectionAttemptsDone, Settings.SourceId, "");
- Producer = PQLib->CreateRawProducer(Settings, DestroyEventRef, Logger);
- StartFuture = Producer->Start(deadline);
-
- std::weak_ptr<TRetryingProducer> self(shared_from_this());
- PQLib->Subscribe(StartFuture,
- this,
- [self](const NThreading::TFuture<TProducerCreateResponse>& f) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->ProcessStart(f);
- }
- });
-}
-
-TDuration TRetryingProducer::UpdateReconnectionDelay() {
- if (LastReconnectionDelay == TDuration::Zero()) {
- LastReconnectionDelay = Settings.ReconnectionDelay;
- } else {
- LastReconnectionDelay *= 2;
- }
- LastReconnectionDelay = Min(LastReconnectionDelay, Settings.MaxReconnectionDelay);
- return LastReconnectionDelay;
-}
-
-void TRetryingProducer::ScheduleRecreation() {
- if (Stopping) {
- return;
- }
- NeedRecreation = true;
- if (!ReconnectionCallback) {
- const TDuration delay = UpdateReconnectionDelay();
- DEBUG_LOG("Schedule subproducer recreation through " << delay, Settings.SourceId, "");
- std::weak_ptr<TRetryingProducer> self(shared_from_this());
- ReconnectionCallback =
- PQLib->GetScheduler().Schedule(delay,
- this,
- [self, sourceId = Settings.SourceId, logger = Logger] {
- auto selfShared = self.lock();
- WRITE_LOG("Subproducer recreation callback. self is " << (selfShared ? "OK" : "nullptr") << ", stopping: " << (selfShared ? selfShared->Stopping : false), sourceId, "", TLOG_DEBUG, logger);
- if (selfShared) {
- selfShared->ReconnectionCallback = nullptr;
- if (!selfShared->Stopping) {
- selfShared->RecreateProducer(TInstant::Now() + selfShared->Settings.StartSessionTimeout);
- }
- }
- });
- }
-}
-
-void TRetryingProducer::ProcessStart(const NThreading::TFuture<TProducerCreateResponse>& f) noexcept {
- INFO_LOG("Subproducer start response: " << f.GetValue().Response, Settings.SourceId, "");
- if (Stopping)
- return;
- if (NeedRecreation)
- return;
- if (!StartFuture.HasValue())
- return;
- if (StartFuture.GetValue().Response.HasError()) {
- WARN_LOG("Subproducer start error: " << f.GetValue().Response, Settings.SourceId, "");
- ScheduleRecreation();
- } else {
- LastReconnectionDelay = TDuration::Zero();
- ReconnectionAttemptsDone = 0;
-
- // recreate on dead
- DEBUG_LOG("Subscribe on subproducer death", Settings.SourceId, "");
- std::weak_ptr<TRetryingProducer> self(shared_from_this());
- PQLib->Subscribe(Producer->IsDead(),
- this,
- [self](const NThreading::TFuture<TError>& error) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->OnProducerDead(error.GetValue());
- }
- });
-
- if (!StartPromise.HasValue()) {
- StartPromise.SetValue(StartFuture.GetValue());
- }
-
- SendData();
- }
-}
-
-void TRetryingProducer::OnProducerDead(const TError& error) {
- WARN_LOG("Subproducer is dead: " << error, Settings.SourceId, "");
- ScheduleRecreation();
-}
-
-void TRetryingProducer::SendData() noexcept {
- if (Stopping)
- return;
- ui64 maxSeqNo = StartFuture.GetValue().Response.GetInit().GetMaxSeqNo();
- std::deque<NThreading::TPromise<TProducerCommitResponse>> promises;
- std::deque<TWriteData> values;
- ui64 prevSeqNo = 0;
- for (auto& d : ResendRequests) {
- Y_VERIFY(d.SeqNo == 0 || d.SeqNo > prevSeqNo);
- if (d.SeqNo != 0) {
- prevSeqNo = d.SeqNo;
- }
- if (d.SeqNo != 0 && d.SeqNo <= maxSeqNo) {
- promises.push_back(Promises.front());
- Promises.pop_front();
- values.push_back(d);
- } else {
- DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data));
- }
- }
- //Requests can be not checked - it is up to client
- for (auto& d : Requests) {
- DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data));
- }
- ResendRequests.clear();
- Requests.clear();
- for (ui32 i = 0; i < promises.size(); ++i) {
- TWriteResponse res;
- res.MutableAck()->SetAlreadyWritten(true);
- res.MutableAck()->SetSeqNo(values[i].SeqNo);
- promises[i].SetValue(TProducerCommitResponse(values[i].SeqNo, std::move(values[i].Data), std::move(res)));
- }
-}
-
-void TRetryingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept {
- Y_VERIFY(data.IsEncoded());
- if (!StartFuture.Initialized()) {
- TWriteResponse res;
- res.MutableError()->SetDescription("producer is not ready");
- res.MutableError()->SetCode(NErrorCode::ERROR);
- promise.SetValue(TProducerCommitResponse{seqNo, std::move(data), std::move(res)});
- return;
- }
-
- Promises.push_back(promise);
-
- if (!StartFuture.HasValue() || NeedRecreation || Stopping || !Requests.empty() || !ResendRequests.empty()) {
- Requests.emplace_back(seqNo, std::move(data));
- } else {
- DelegateWriteAndSubscribe(seqNo, std::move(data));
- }
-}
-
-void TRetryingProducer::DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept {
- Y_VERIFY(InFlightRequests.size() == Futures.size());
- if (seqNo == 0) {
- Futures.push_back(Producer->Write(data));
- } else {
- Futures.push_back(Producer->Write(seqNo, data));
- }
- InFlightRequests.emplace_back(seqNo, std::move(data));
-
- std::weak_ptr<TRetryingProducer> self(shared_from_this());
- PQLib->Subscribe(Futures.back(),
- this,
- [self](const NThreading::TFuture<TProducerCommitResponse>&) {
- auto selfShared = self.lock();
- if (selfShared) {
- selfShared->ProcessFutures();
- }
- });
-}
-
-NThreading::TFuture<TError> TRetryingProducer::IsDead() noexcept {
- return IsDeadPromise.GetFuture();
-}
-
-void TRetryingProducer::ProcessFutures() noexcept {
- NThreading::TPromise<TProducerCommitResponse> promise;
- NThreading::TFuture<TProducerCommitResponse> future;
- {
- if (Stopping) {
- return;
- }
- ++ToProcess;
- while (ToProcess) {
- if (Futures.empty() || !Futures.front().HasValue()) {
- break;
- }
-
- Y_VERIFY(InFlightRequests.size() == Futures.size());
- --ToProcess;
-
- const TProducerCommitResponse& response = Futures.front().GetValue();
- const TWriteData& writeData = InFlightRequests.front();
- Y_VERIFY(Promises.size() == Futures.size() + Requests.size() + ResendRequests.size());
-
- if (NeedRecreation) {
- ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data));
- InFlightRequests.pop_front();
- Futures.pop_front();
- continue;
- }
-
- if (Futures.front().GetValue().Response.HasError()) {
- Y_VERIFY(!NeedRecreation);
- ScheduleRecreation();
- WARN_LOG("Future response with error: " << response.Response, Settings.SourceId, "");
- ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data));
- InFlightRequests.pop_front();
- Futures.pop_front();
- Producer = nullptr;
- continue;
- }
-
- promise = Promises.front();
- Promises.pop_front();
- future = Futures.front();
- InFlightRequests.pop_front();
- Futures.pop_front();
- promise.SetValue(future.GetValue());
- }
- if (NeedRecreation && Futures.empty() && !ReconnectionCallback) {
- // We need recreation, but scheduled recreation hasn't start producer because of nonempty future list.
- DEBUG_LOG("Recreating subproducer after all futures were processed", Settings.SourceId, "");
- RecreateProducer(TInstant::Now() + Settings.StartSessionTimeout);
- }
- }
-}
-
-void TRetryingProducer::Destroy(const TString& description) {
- TError error;
- error.SetDescription(description);
- error.SetCode(NErrorCode::ERROR);
- Destroy(error);
-}
-
-void TRetryingProducer::SubscribeDestroyed() {
- NThreading::TPromise<void> promise = ProducersDestroyed;
- auto handler = [promise](const auto&) mutable {
- promise.SetValue();
- };
- if (Producer) {
- WaitExceptionOrAll(DestroyedPromise.GetFuture(), Producer->Destroyed())
- .Subscribe(handler);
- } else {
- DestroyedPromise.GetFuture()
- .Subscribe(handler);
- }
-
- DestroyPQLibRef();
-}
-
-void TRetryingProducer::Destroy(const TError& error) {
- if (Stopping) {
- return;
- }
- Stopping = true;
- SubscribeDestroyed();
- const bool started = StartFuture.Initialized();
- Producer = nullptr;
- if (started) {
- Y_VERIFY(Promises.size() == ResendRequests.size() + Requests.size() + InFlightRequests.size());
- ResendRequests.insert(ResendRequests.begin(), InFlightRequests.begin(), InFlightRequests.end());
- ResendRequests.insert(ResendRequests.end(), Requests.begin(), Requests.end());
- for (auto& v : ResendRequests) {
- TWriteResponse resp;
- *resp.MutableError() = error;
- Promises.front().SetValue(TProducerCommitResponse(v.SeqNo, std::move(v.Data), std::move(resp)));
- Promises.pop_front();
- }
- Y_VERIFY(Promises.empty());
- }
- if (StartPromise.Initialized() && !StartPromise.HasValue()) {
- TWriteResponse resp;
- *resp.MutableError() = error;
- StartPromise.SetValue(TProducerCreateResponse(std::move(resp)));
- }
- IsDeadPromise.SetValue(error);
-}
-
-NThreading::TFuture<void> TRetryingProducer::Destroyed() noexcept {
- return ProducersDestroyed.GetFuture();
-}
-
-void TRetryingProducer::OnStartDeadline() {
- if (!StartPromise.HasValue()) {
- TError error;
- error.SetDescription("Start timeout.");
- error.SetCode(NErrorCode::CREATE_TIMEOUT);
- Destroy(error);
- }
-}
-
-void TRetryingProducer::Cancel() {
- Destroy(GetCancelReason());
-}
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h
deleted file mode 100644
index f1242564b9..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#pragma once
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include "scheduler.h"
-#include "internals.h"
-#include "persqueue_p.h"
-#include "producer.h"
-#include "iproducer_p.h"
-#include <library/cpp/threading/future/future.h>
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-#include <deque>
-
-namespace NPersQueue {
-
-class TRetryingProducer: public IProducerImpl, public std::enable_shared_from_this<TRetryingProducer> {
-public:
- using IProducerImpl::Write;
- // If seqno == 0, we assume it to be autodefined
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override;
- void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override {
- TRetryingProducer::Write(promise, 0, std::move(data));
- }
-
- TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib,
- TIntrusivePtr<ILogger> logger);
-
- NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override;
-
- NThreading::TFuture<TError> IsDead() noexcept override;
-
- NThreading::TFuture<void> Destroyed() noexcept override;
-
- ~TRetryingProducer() noexcept;
-
-
- void RecreateProducer(TInstant deadline) noexcept;
- void ProcessStart(const NThreading::TFuture<TProducerCreateResponse>&) noexcept;
- void SendData() noexcept;
- void ProcessFutures() noexcept;
- void DoRecreate(TInstant deadline) noexcept;
- void DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept;
- TDuration UpdateReconnectionDelay();
- void ScheduleRecreation();
- void Destroy(const TError& error);
- void Destroy(const TString& description); // the same but with Code=ERROR
- void SubscribeDestroyed();
- void OnStartDeadline();
- void OnProducerDead(const TError& error);
-
- void Cancel() override;
-
-protected:
- TProducerSettings Settings;
- TIntrusivePtr<ILogger> Logger;
- std::shared_ptr<IProducerImpl> Producer;
- std::deque<TWriteData> Requests;
- std::deque<TWriteData> ResendRequests;
- std::deque<TWriteData> InFlightRequests;
- std::deque<NThreading::TPromise<TProducerCommitResponse>> Promises;
- std::deque<NThreading::TFuture<TProducerCommitResponse>> Futures;
- NThreading::TFuture<TProducerCreateResponse> StartFuture;
- NThreading::TPromise<TProducerCreateResponse> StartPromise;
- NThreading::TPromise<TError> IsDeadPromise;
- bool NeedRecreation;
- bool Stopping;
- ui32 ToProcess;
- TDuration LastReconnectionDelay;
- unsigned ReconnectionAttemptsDone;
- TIntrusivePtr<TScheduler::TCallbackHandler> ReconnectionCallback;
- NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>();
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp
deleted file mode 100644
index 3fdb17d954..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp
+++ /dev/null
@@ -1,355 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
-namespace NPersQueue {
-
-Y_UNIT_TEST_SUITE(TRetryingProducerTest) {
- TProducerSettings FakeSettings() {
- TProducerSettings settings;
- settings.ReconnectOnFailure = true;
- settings.Server = TServerSetting{"localhost"};
- settings.Topic = "topic";
- settings.SourceId = "src";
- return settings;
- }
-
- Y_UNIT_TEST(NotStartedProducerCanBeDestructed) {
- // Test that producer doesn't hang on till shutdown
- TPQLib lib;
- TProducerSettings settings = FakeSettings();
- lib.CreateProducer(settings, {}, false);
- }
-
- Y_UNIT_TEST(StartDeadlineExpires) {
- if (std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"))
- return;
- TPQLibSettings libSettings;
- libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1);
- libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG);
- TPQLib lib(libSettings);
- THolder<IProducer> producer;
- {
- TProducerSettings settings = FakeSettings();
- settings.ReconnectionDelay = TDuration::MilliSeconds(1);
- settings.StartSessionTimeout = TDuration::MilliSeconds(10);
- producer = lib.CreateProducer(settings, {}, false);
- }
- const TInstant beforeStart = TInstant::Now();
- auto future = producer->Start(TDuration::MilliSeconds(100));
- UNIT_ASSERT(future.GetValueSync().Response.HasError());
- UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT,
- "Error: " << future.GetValueSync().Response.GetError());
- const TInstant now = TInstant::Now();
- UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now);
-
- DestroyAndWait(producer);
- }
-
- Y_UNIT_TEST(StartMaxAttemptsExpire) {
- TPQLibSettings libSettings;
- libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1);
- TPQLib lib(libSettings);
- THolder<IProducer> producer;
- {
- TProducerSettings settings = FakeSettings();
- settings.ReconnectionDelay = TDuration::MilliSeconds(10);
- settings.StartSessionTimeout = TDuration::MilliSeconds(10);
- settings.MaxAttempts = 3;
- producer = lib.CreateProducer(settings, {}, false);
- }
- const TInstant beforeStart = TInstant::Now();
- auto future = producer->Start();
- UNIT_ASSERT(future.GetValueSync().Response.HasError());
- const TInstant now = TInstant::Now();
- UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now);
-
- DestroyAndWait(producer);
- }
-
- void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response);
- }
-
- void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) {
- const TProducerCommitResponse& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response);
- }
-
- void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response);
- }
-
- void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) {
- for (ui32 i = 0; i < batch.MessageSize(); ++i) {
- ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo();
- UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i);
- }
- }
-
- void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response);
- }
-
- void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response);
- }
-
- void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) {
- const TConsumerMessage& resp = respFuture.GetValueSync();
- UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response);
- }
-
- TProducerSettings MakeProducerSettings(const TTestServer& testServer) {
- TProducerSettings producerSettings;
- producerSettings.ReconnectOnFailure = true;
- producerSettings.Topic = "topic1";
- producerSettings.SourceId = "123";
- producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- producerSettings.Codec = ECodec::LZOP;
- producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10);
- return producerSettings;
- }
-
- TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) {
- TConsumerSettings consumerSettings;
- consumerSettings.ClientId = "user";
- consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort};
- consumerSettings.Topics.push_back("topic1");
- return consumerSettings;
- }
-
- Y_UNIT_TEST(ReconnectsToServer) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- const size_t partitions = 10;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib PQLib;
-
- auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false);
- producer->Start().Wait();
- TFuture<TError> isDead = producer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false);
- consumer->Start().Wait();
-
- auto read1 = consumer->GetNextMessage();
-
- // write first value
- auto write1 = producer->Write(1, TString("blob1"));
- AssertWriteValid(write1);
- AssertReadValid(read1);
-
- // commit
- consumer->Commit({1});
- auto commitAck = consumer->GetNextMessage();
- AssertCommited(commitAck);
-
- auto read2 = consumer->GetNextMessage();
- testServer.ShutdownServer();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto write2 = producer->Write(2, TString("blob2"));
-
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
-
- AssertWriteValid(write2);
-
- AssertReadFailed(read2);
- consumer->IsDead().Wait();
-
- consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false);
- consumer->Start().Wait();
-
- read2 = consumer->GetNextMessage();
- AssertReadValid(read2);
-
- UNIT_ASSERT(!isDead.HasValue());
-
- DestroyAndWait(producer);
- DestroyAndWait(consumer);
- }
-
- static void DiesOnTooManyReconnectionAttempts(bool callWrite) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2);
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib PQLib;
-
- TProducerSettings settings = MakeProducerSettings(testServer);
- settings.MaxAttempts = 3;
- settings.ReconnectionDelay = TDuration::MilliSeconds(100);
- auto producer = PQLib.CreateProducer(settings, logger, false);
- producer->Start().Wait();
- TFuture<TError> isDead = producer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- // shutdown server
- const TInstant beforeShutdown = TInstant::Now();
- testServer.ShutdownServer();
-
- NThreading::TFuture<TProducerCommitResponse> write;
- if (callWrite) {
- write = producer->Write(TString("data"));
- }
-
- isDead.Wait();
- const TInstant afterDead = TInstant::Now();
- // 3 attempts: 100ms, 200ms and 400ms
- UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(700), "real difference: " << (afterDead - beforeShutdown));
-
- if (callWrite) {
- AssertWriteFailed(write);
- }
- }
-
- Y_UNIT_TEST(WriteDataThatWasSentBeforeConnectToServer) {
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer(false);
-
- testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- const size_t partitions = 10;
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
-
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLib PQLib;
-
- auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false);
- auto producerStarted = producer->Start();
- auto write1 = producer->Write(1, TString("a"));
- producerStarted.Wait();
- TVector<TFuture<TProducerCommitResponse>> acks;
- for (int i = 2; i <= 1000; ++i) {
- acks.push_back(producer->Write(i, TString("b")));
- }
-
- AssertWriteValid(write1);
- for (auto& ack: acks) {
- ack.Wait();
- }
-
- auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false);
- consumer->Start().Wait();
-
- ui64 readMessages = 0;
- while (readMessages < 1000) {
- auto read1 = consumer->GetNextMessage();
- AssertReadValid(read1);
- auto& batch = read1.GetValueSync().Response.GetData().GetMessageBatch(0);
- AssertReadContinuous(batch, readMessages + 1);
- readMessages += batch.MessageSize();
- }
-
- DestroyAndWait(producer);
- DestroyAndWait(consumer);
- }
-
- Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutWrite) {
- // Check that we reconnect even without explicit write errors
- DiesOnTooManyReconnectionAttempts(false);
- }
-
- Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithWrite) {
- DiesOnTooManyReconnectionAttempts(true);
- }
-
- Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) {
- return; // Test is ignored. FIX: KIKIMR-7886
- TTestServer testServer(false);
- testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10));
- testServer.StartServer();
-
- const size_t partitions = 1;
- testServer.AnnoyingClient->InitRoot();
- testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC);
- testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions);
- testServer.AnnoyingClient->InitSourceIds();
-
- testServer.WaitInit("topic1");
-
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false);
- UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError());
- TFuture<TError> isDead = producer->IsDead();
- UNIT_ASSERT(!isDead.HasValue());
-
- testServer.ShutdownServer();
- UNIT_ASSERT(!isDead.HasValue());
-
- auto write1 = producer->Write(1, TString("blob1"));
- auto write2 = producer->Write(2, TString("blob2"));
-
- UNIT_ASSERT(!write1.HasValue());
- UNIT_ASSERT(!write2.HasValue());
-
- PQLib = nullptr;
-
- UNIT_ASSERT(write1.HasValue());
- UNIT_ASSERT(write2.HasValue());
-
- UNIT_ASSERT(write1.GetValue().Response.HasError());
- UNIT_ASSERT(write2.GetValue().Response.HasError());
-
- auto write3 = producer->Write(3, TString("blob3"));
- UNIT_ASSERT(write3.HasValue());
- UNIT_ASSERT(write3.GetValue().Response.HasError());
- }
-
- Y_UNIT_TEST(CancelsStartAfterPQLibDeath) {
- TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- TPQLibSettings pqLibSettings;
- pqLibSettings.DefaultLogger = logger;
- THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings);
-
- auto producer = PQLib->CreateProducer(FakeSettings(), logger, false);
- auto start = producer->Start();
- UNIT_ASSERT(!start.HasValue());
-
- PQLib = nullptr;
-
- UNIT_ASSERT(start.HasValue());
- UNIT_ASSERT(start.GetValue().Response.HasError());
-
- auto dead = producer->IsDead();
- UNIT_ASSERT(dead.HasValue());
- dead.GetValueSync();
-
- auto write = producer->Write(1, TString("blob1"));
- UNIT_ASSERT(write.HasValue());
- UNIT_ASSERT(write.GetValueSync().Response.HasError());
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp
deleted file mode 100644
index ba6c4ebb40..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-#include "persqueue_p.h"
-#include "scheduler.h"
-
-#include <util/system/thread.h>
-
-namespace NPersQueue {
-void TScheduler::TCallbackHandler::Execute() {
- auto guard = Guard(Lock);
- if (Callback) {
- Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(std::move(Callback)));
- Callback = nullptr;
- }
-}
-
-void TScheduler::TCallbackHandler::TryCancel() {
- auto guard = Guard(Lock);
- if (Callback) {
- Callback = nullptr;
- }
-}
-
-bool TScheduler::TCallbackHandlersCompare::operator()(const TIntrusivePtr<TScheduler::TCallbackHandler>& h1, const TIntrusivePtr<TScheduler::TCallbackHandler>& h2) const {
- return h1->Time > h2->Time;
-}
-
-TScheduler::TScheduler(TPQLibPrivate* pqLib)
- : Shutdown(false)
- , PQLib(pqLib)
-{
- Thread = SystemThreadFactory()->Run([this] {
- this->SchedulerThread();
- });
-}
-
-TScheduler::~TScheduler() {
- ShutdownAndWait();
-}
-
-void TScheduler::AddToSchedule(TIntrusivePtr<TCallbackHandler> handler) {
- {
- auto guard = Guard(Lock);
- Callbacks.push(std::move(handler));
- }
- Event.Signal();
-}
-
-void TScheduler::ShutdownAndWait() {
- AtomicSet(Shutdown, true);
- Event.Signal();
- Thread->Join();
-}
-
-void TScheduler::SchedulerThread() {
- TThread::SetCurrentThreadName("pqlib_scheduler");
- while (!AtomicGet(Shutdown)) {
- TInstant deadline = TInstant::Max();
- std::vector<TIntrusivePtr<TCallbackHandler>> callbacks;
- {
- // define next deadline and get expired callbacks
- auto guard = Guard(Lock);
- const TInstant now = TInstant::Now();
- while (!Callbacks.empty()) {
- const auto& top = Callbacks.top();
- if (top->Time <= now) {
- callbacks.push_back(top);
- Callbacks.pop();
- } else {
- deadline = top->Time;
- break;
- }
- }
- }
-
- // execute callbacks
- bool shutdown = false;
- for (auto& callback : callbacks) {
- if (shutdown) {
- callback->TryCancel();
- } else {
- callback->Execute();
- }
- shutdown = shutdown || AtomicGet(Shutdown);
- }
-
- if (!shutdown) {
- Event.WaitD(deadline);
- }
- }
-
- // cancel all callbacks and clear data
- while (!Callbacks.empty()) {
- Callbacks.top()->TryCancel();
- Callbacks.pop();
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h
deleted file mode 100644
index c90ab87795..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h
+++ /dev/null
@@ -1,88 +0,0 @@
-#pragma once
-#include <util/datetime/base.h>
-#include <util/generic/ptr.h>
-#include <util/system/event.h>
-#include <util/system/spinlock.h>
-#include <util/thread/factory.h>
-
-#include <functional>
-#include <queue>
-#include <utility>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class TScheduler {
- struct TCallbackHandlersCompare;
-public:
- using TCallback = std::function<void()>;
-
- class TCallbackHandler: public TAtomicRefCount<TCallbackHandler> {
- friend class TScheduler;
- friend struct TCallbackHandlersCompare;
-
- template <class TFunc>
- TCallbackHandler(TInstant time, TFunc&& func, TPQLibPrivate* pqLib, const void* queueTag)
- : Time(time)
- , Callback(std::forward<TFunc>(func))
- , PQLib(pqLib)
- , QueueTag(queueTag)
- {
- }
-
- void Execute();
-
- public:
- // Cancels execution of callback.
- // If callback is already canceled or executes (executed), does nothing.
- // Posteffect: callback is guaranteed to be destroyed after this call.
- void TryCancel();
-
- private:
- TInstant Time;
- TCallback Callback;
- TAdaptiveLock Lock;
- TPQLibPrivate* PQLib;
- const void* QueueTag;
- };
-
-public:
- // Starts a scheduler thread.
- explicit TScheduler(TPQLibPrivate* pqLib);
-
- // Stops a scheduler thread.
- ~TScheduler();
-
- // Schedules a new callback to be executed
- template <class TFunc>
- TIntrusivePtr<TCallbackHandler> Schedule(TInstant time, const void* queueTag, TFunc&& func) {
- TIntrusivePtr<TCallbackHandler> handler(new TCallbackHandler(time, std::forward<TFunc>(func), PQLib, queueTag));
- AddToSchedule(handler);
- return handler;
- }
-
- template <class TFunc>
- TIntrusivePtr<TCallbackHandler> Schedule(TDuration delta, const void* queueTag, TFunc&& func) {
- return Schedule(TInstant::Now() + delta, queueTag, std::forward<TFunc>(func));
- }
-
-private:
- void AddToSchedule(TIntrusivePtr<TCallbackHandler> handler);
- void ShutdownAndWait();
- void SchedulerThread();
-
-private:
- struct TCallbackHandlersCompare {
- bool operator()(const TIntrusivePtr<TCallbackHandler>& h1, const TIntrusivePtr<TCallbackHandler>& h2) const;
- };
-
- using TCallbackQueue = std::priority_queue<TIntrusivePtr<TCallbackHandler>, std::vector<TIntrusivePtr<TCallbackHandler>>, TCallbackHandlersCompare>;
- TAdaptiveLock Lock;
- TAutoEvent Event;
- TCallbackQueue Callbacks;
- TAtomic Shutdown;
- THolder<IThreadFactory::IThread> Thread;
- TPQLibPrivate* PQLib;
-};
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp
deleted file mode 100644
index 83ce25c6b4..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-#include "scheduler.h"
-#include "persqueue_p.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/event.h>
-
-namespace NPersQueue {
-const void* Tag = (const void*)42;
-Y_UNIT_TEST_SUITE(TSchedulerTest) {
- Y_UNIT_TEST(EmptySchedulerCanDestruct) {
- // Test that scheduler doesn't hang on till shutdown
- TScheduler scheduler(nullptr);
- }
-
- Y_UNIT_TEST(ExecutesInProperOrder) {
- TSystemEvent ev;
- bool callback1Executed = false;
- TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({}));
- TScheduler scheduler(lib.Get());
- scheduler.Schedule(TDuration::MilliSeconds(100), Tag, [&callback1Executed, &ev] {
- UNIT_ASSERT(callback1Executed);
- ev.Signal();
- });
- scheduler.Schedule(TDuration::MilliSeconds(50), Tag, [&callback1Executed] {
- callback1Executed = true;
- });
- ev.Wait();
- }
-
- Y_UNIT_TEST(CancelsAndClearsData) {
- TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({}));
- TScheduler scheduler(lib.Get());
- std::shared_ptr<TString> ptr(new TString());
- bool callbackExecuted = false;
- auto h = scheduler.Schedule(TDuration::Seconds(50), Tag, [&callbackExecuted, ptr] {
- callbackExecuted = true;
- });
- UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 2);
- h->TryCancel();
- UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1);
- UNIT_ASSERT(!callbackExecuted);
- Sleep(TDuration::MilliSeconds(51));
- UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1);
- UNIT_ASSERT(!callbackExecuted);
- }
-
- Y_UNIT_TEST(ExitsThreadImmediately) {
- TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({}));
- std::shared_ptr<TString> ptr(new TString());
- bool callback1Executed = false;
- bool callback2Executed = false;
- TSystemEvent ev, ev2;
- auto now = TInstant::Now();
- lib->GetScheduler().Schedule(TDuration::Seconds(500), Tag, [&callback1Executed, ptr] {
- callback1Executed = true;
- });
- lib->GetScheduler().Schedule(now, Tag, [&callback2Executed, &ev, &ev2, ptr] {
- callback2Executed = true;
- ev2.Wait();
- ev.Signal();
- });
- UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 3);
- ev2.Signal();
-
- // kill scheduler
- ev.Wait();
- lib.Reset();
-
- UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1);
- UNIT_ASSERT(!callback1Executed);
- UNIT_ASSERT(callback2Executed);
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp
deleted file mode 100644
index 4fc6a1c7b4..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-#include <library/cpp/streams/lzop/lzop.h>
-#include <library/cpp/streams/zstd/zstd.h>
-
-#include <util/generic/store_policy.h>
-#include <util/generic/utility.h>
-#include <util/stream/str.h>
-#include <util/stream/zlib.h>
-
-namespace NPersQueue {
-
-const TString& TServerSetting::GetRootDatabase() {
- static const TString RootDatabase = "/Root";
- return RootDatabase;
-}
-
-class TZLibToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZLibCompress {
-public:
- TZLibToStringCompressor(TString& dst, ZLib::StreamType type, size_t quality)
- : TEmbedPolicy<TStringOutput>(dst)
- , TZLibCompress(TEmbedPolicy::Ptr(), type, quality)
- {
- }
-};
-
-class TLzopToStringCompressor: private TEmbedPolicy<TStringOutput>, public TLzopCompress {
-public:
- TLzopToStringCompressor(TString& dst)
- : TEmbedPolicy<TStringOutput>(dst)
- , TLzopCompress(TEmbedPolicy::Ptr())
- {
- }
-};
-
-class TZstdToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZstdCompress {
-public:
- TZstdToStringCompressor(TString& dst, int quality)
- : TEmbedPolicy<TStringOutput>(dst)
- , TZstdCompress(TEmbedPolicy::Ptr(), quality)
- {
- }
-};
-
-
-TData TData::Encode(TData source, ECodec defaultCodec, int quality) {
- Y_VERIFY(!source.Empty());
- TData data = std::move(source);
- if (data.IsEncoded()) {
- return data;
- }
- Y_VERIFY(defaultCodec != ECodec::RAW && defaultCodec != ECodec::DEFAULT);
- if (data.Codec == ECodec::DEFAULT) {
- data.Codec = defaultCodec;
- }
- THolder<IOutputStream> coder = CreateCoder(data.Codec, data, quality);
- coder->Write(data.SourceData);
- coder->Finish(); // &data.EncodedData may be already invalid on coder destruction
- return data;
-}
-
-TData TData::MakeRawIfNotEncoded(TData source) {
- Y_VERIFY(!source.Empty());
- TData data = std::move(source);
- if (!data.IsEncoded()) {
- data.EncodedData = data.SourceData;
- data.Codec = ECodec::RAW;
- }
- return data;
-}
-
-THolder<IOutputStream> TData::CreateCoder(ECodec codec, TData& result, int quality) {
- switch (codec) {
- case ECodec::GZIP:
- return MakeHolder<TZLibToStringCompressor>(result.EncodedData, ZLib::GZip, quality >= 0 ? quality : 6);
- case ECodec::LZOP:
- return MakeHolder<TLzopToStringCompressor>(result.EncodedData);
- case ECodec::ZSTD:
- return MakeHolder<TZstdToStringCompressor>(result.EncodedData, quality);
- default:
- Y_FAIL("NOT IMPLEMENTED CODEC TYPE");
- }
-}
-
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp
deleted file mode 100644
index 06c4b28454..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/string/builder.h>
-
-namespace NPersQueue {
-Y_UNIT_TEST_SUITE(TDataTest) {
- ECodec Codecs[] = {
- ECodec::LZOP,
- ECodec::GZIP,
- };
-
- TString DebugString(const TData& data) {
- return TStringBuilder() << "{ Ts: " << data.GetTimestamp().GetValue() << ", SrcData: \"" << data.GetSourceData()
- << "\", Encoded: \"" << (data.IsEncoded() ? data.GetEncodedData() : TString())
- << "\", Codec: " << data.GetCodecType() << " }";
- }
-
- Y_UNIT_TEST(RawDataIsEncoded) {
- auto now = TInstant::Now();
- TData data = TData::Raw("trololo", now);
- TData otherRawData = TData("trololo", ECodec::RAW, now);
- UNIT_ASSERT_C(data.IsEncoded(), "data: " << DebugString(data));
- UNIT_ASSERT_C(!data.Empty(), "data: " << DebugString(data));
- UNIT_ASSERT_EQUAL_C(data.GetCodecType(), ECodec::RAW, "data: " << DebugString(data));
- UNIT_ASSERT_EQUAL_C(data, otherRawData, "data: " << DebugString(data) << ", other: " << DebugString(otherRawData));
- }
-
- Y_UNIT_TEST(EncodedDataIsEncoded) {
- for (ECodec codec : Codecs) {
- TData data = TData::Encoded("trololo", codec);
- UNIT_ASSERT(data.IsEncoded());
- UNIT_ASSERT(!data.Empty());
- UNIT_ASSERT_EQUAL(data.GetCodecType(), codec);
- }
- }
-
- Y_UNIT_TEST(ModifiesState) {
- for (ECodec codec : Codecs) {
- for (ECodec defaultCodec : Codecs) {
- TData data("trololo", codec);
- UNIT_ASSERT(!data.IsEncoded());
- UNIT_ASSERT(!data.Empty());
-
- for (size_t i = 0; i < 2; ++i) {
- data = TData::Encode(std::move(data), defaultCodec, 2); // encode twice is OK
- UNIT_ASSERT(data.IsEncoded());
- UNIT_ASSERT(!data.Empty());
- UNIT_ASSERT(!data.GetEncodedData().empty());
- UNIT_ASSERT_EQUAL(data.GetCodecType(), codec);
- }
- }
- }
- }
-
- Y_UNIT_TEST(HandlesDefaultCodec) {
- for (ECodec defaultCodec : Codecs) {
- TData data = TString("trololo");
- UNIT_ASSERT(!data.IsEncoded());
- UNIT_ASSERT(!data.Empty());
-
- data = TData::Encode(data, defaultCodec, -1);
- UNIT_ASSERT(data.IsEncoded());
- UNIT_ASSERT(!data.Empty());
- UNIT_ASSERT_STRINGS_EQUAL(data.GetSourceData(), "trololo");
- UNIT_ASSERT(!data.GetEncodedData().empty());
- UNIT_ASSERT_EQUAL(data.GetCodecType(), defaultCodec);
- }
- }
-
- Y_UNIT_TEST(MakesRaw) {
- TData data = TString("trololo");
- UNIT_ASSERT(!data.IsEncoded());
- data = TData::MakeRawIfNotEncoded(data);
- UNIT_ASSERT(data.IsEncoded());
- UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::RAW);
- }
-
- Y_UNIT_TEST(DoesNotMakeRaw) {
- TData data = TData::Encoded("trololo", ECodec::GZIP);
- UNIT_ASSERT(data.IsEncoded());
- data = TData::MakeRawIfNotEncoded(data);
- UNIT_ASSERT(data.IsEncoded());
- UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::GZIP);
- }
-}
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp
deleted file mode 100644
index a7d65ddc88..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-#include "validate_grpc_metadata.h"
-#include <contrib/libs/grpc/include/grpc/grpc.h>
-#include <util/generic/string.h>
-#include <util/string/escape.h>
-#include <util/string/builder.h>
-
-namespace NGrpc {
- bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error) {
- error.clear();
- grpc_slice keySlice = grpc_slice_from_static_buffer(key.c_str(), key.size());
- int ok = grpc_header_key_is_legal(keySlice);
- if (!ok) {
- error = TStringBuilder() << "gRPC metadata header key is illegal: \"" << EscapeC(key) << "\"";
- return false;
- }
- grpc_slice valueSlice = grpc_slice_from_static_buffer(value.c_str(), value.size());
- ok = grpc_is_binary_header(keySlice) || grpc_header_nonbin_value_is_legal(valueSlice);
- if (!ok) {
- error = TStringBuilder() << "gRPC metadata header value with key \"" << key << "\" is illegal: \"" << EscapeC(value) << "\"";
- return false;
- }
- return true;
- }
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h
deleted file mode 100644
index 43fd82cef0..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#pragma once
-#include <util/generic/fwd.h>
-
-namespace NGrpc {
- // Validates gRPC metadata header key and value. Returns 'true' if validations fails, otherwise returns 'false' and sets 'error' value.
- // For more information see 'Custom-Metadata' section in gRPC over HTTP2 (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md).
- // Note that in case of authentication data validation 'error' may contain sensitive information.
- bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error);
-
- // Creates hexadecimal representation of 's' in format '{0x01, 0x02, 0x03}'
- TString ToHexString(const TString& s);
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp
deleted file mode 100644
index bbda57f538..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "validate_grpc_metadata.h"
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NGrpc;
-
-Y_UNIT_TEST_SUITE(NValidateGrpcMetadata) {
- Y_UNIT_TEST(ValidateHeaderIsLegal) {
- TString error = "error";
- UNIT_ASSERT_C(ValidateHeaderIsLegal("key", "value", error), error);
- // Assert 'error' is cleared
- UNIT_ASSERT_C(error.empty(), error);
-
- // Valid character values upper and lower bounds
- UNIT_ASSERT_C(ValidateHeaderIsLegal("\x30\x39\x61\x7A_-.", "\x20\x7E", error), error);
- UNIT_ASSERT_C(error.empty(), error);
-
- TString null = " ";
- null[0] = '\0';
- UNIT_ASSERT(!ValidateHeaderIsLegal("key", null, error));
- UNIT_ASSERT_C(error.Contains("\\0"), error);
- Cerr << "Error is '" << error << "'" << Endl;
-
- // Simple escape sequences
- UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x0A\t", error));
- UNIT_ASSERT_C(error.Contains("\\n\\t"), error);
- Cerr << "Error is '" << error << "'" << Endl;
-
- UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x1F", error));
- UNIT_ASSERT_C(error.Contains("\\x1F"), error);
- Cerr << "Error is '" << error << "'" << Endl;
-
- UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x7F", error));
- UNIT_ASSERT_C(error.Contains("\\x7F"), error);
- Cerr << "Error is '" << error << "'" << Endl;
-
- // Octal character
- UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\177", error));
- UNIT_ASSERT_C(error.Contains("\\x7F"), error);
- Cerr << "Error is '" << error << "'" << Endl;
-
- // Invalid header key
- UNIT_ASSERT(!ValidateHeaderIsLegal("key\n", "value", error));
- UNIT_ASSERT_C(error.Contains("\\n"), error);
- Cerr << "Error is '" << error << "'" << Endl;
- }
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp
deleted file mode 100644
index b679eda9f4..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp
+++ /dev/null
@@ -1,561 +0,0 @@
-#include "ydb_sdk_consumer.h"
-#include "persqueue_p.h"
-
-#include <ydb/library/persqueue/topic_parser_public/topic_parser.h>
-
-#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h>
-#include <util/generic/is_in.h>
-#include <util/string/builder.h>
-
-namespace NPersQueue {
-
-static TError MakeError(const TString& description, NErrorCode::EErrorCode code) {
- TError error;
- error.SetDescription(description);
- error.SetCode(code);
- return error;
-}
-
-static TString MakeLegacyTopicName(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStream) {
- // rt3.man--account--topic
- return BuildFullTopicName(partitionStream->GetTopicPath(), partitionStream->GetCluster());
-}
-
-static NErrorCode::EErrorCode ToErrorCode(NYdb::EStatus status) {
- switch (status) {
- case NYdb::EStatus::STATUS_UNDEFINED:
- return NErrorCode::ERROR;
- case NYdb::EStatus::SUCCESS:
- return NErrorCode::OK;
- case NYdb::EStatus::BAD_REQUEST:
- return NErrorCode::BAD_REQUEST;
- case NYdb::EStatus::UNAUTHORIZED:
- return NErrorCode::ACCESS_DENIED;
- case NYdb::EStatus::INTERNAL_ERROR:
- return NErrorCode::ERROR;
- case NYdb::EStatus::ABORTED:
- [[fallthrough]];
- case NYdb::EStatus::UNAVAILABLE:
- return NErrorCode::ERROR;
- case NYdb::EStatus::OVERLOADED:
- return NErrorCode::OVERLOAD;
- case NYdb::EStatus::SCHEME_ERROR:
- return NErrorCode::UNKNOWN_TOPIC;
- case NYdb::EStatus::GENERIC_ERROR:
- [[fallthrough]];
- case NYdb::EStatus::TIMEOUT:
- [[fallthrough]];
- case NYdb::EStatus::BAD_SESSION:
- [[fallthrough]];
- case NYdb::EStatus::PRECONDITION_FAILED:
- [[fallthrough]];
- case NYdb::EStatus::ALREADY_EXISTS:
- [[fallthrough]];
- case NYdb::EStatus::NOT_FOUND:
- [[fallthrough]];
- case NYdb::EStatus::SESSION_EXPIRED:
- [[fallthrough]];
- case NYdb::EStatus::CANCELLED:
- [[fallthrough]];
- case NYdb::EStatus::UNDETERMINED:
- [[fallthrough]];
- case NYdb::EStatus::UNSUPPORTED:
- [[fallthrough]];
- case NYdb::EStatus::SESSION_BUSY:
- [[fallthrough]];
- case NYdb::EStatus::TRANSPORT_UNAVAILABLE:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_RESOURCE_EXHAUSTED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_DEADLINE_EXCEEDED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_INTERNAL_ERROR:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_CANCELLED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_UNAUTHENTICATED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_CALL_UNIMPLEMENTED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_OUT_OF_RANGE:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_DISCOVERY_FAILED:
- [[fallthrough]];
- case NYdb::EStatus::CLIENT_LIMITS_REACHED:
- return NErrorCode::ERROR;
- }
-}
-
-TYdbSdkCompatibilityConsumer::TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings,
- std::shared_ptr<void> destroyEventRef,
- TIntrusivePtr<TPQLibPrivate> pqLib,
- TIntrusivePtr<ILogger> logger,
- NYdb::NPersQueue::TPersQueueClient& client)
- : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib))
- , Settings(settings)
- , Logger(std::move(logger))
- , Client(client)
-{
-}
-
-TYdbSdkCompatibilityConsumer::~TYdbSdkCompatibilityConsumer() {
- Destroy("Destructor called");
-}
-
-void TYdbSdkCompatibilityConsumer::Init() {
- YdbSdkSettings = MakeReadSessionSettings(); // weak_from_this() is used here.
-}
-
-static const THashSet<TString> FederationClusters = {
- "iva",
- "man",
- "myt",
- "sas",
- "vla",
-};
-
-static TString FindClusterFromEndpoint(const TString& endpoint) {
- size_t pos = endpoint.find(".logbroker.yandex.net");
- if (pos == TString::npos) {
- pos = endpoint.find(".logbroker-prestable.yandex.net");
- if (pos == TString::npos) {
- return {}; // something strange or cross dc cloud cluster.
- }
- }
- TString prefix = endpoint.substr(0, pos);
- if (IsIn(FederationClusters, prefix)) {
- return prefix;
- }
- return {}; // no cluster in cross dc federation.
-}
-
-NYdb::NPersQueue::TReadSessionSettings TYdbSdkCompatibilityConsumer::MakeReadSessionSettings() {
- NYdb::NPersQueue::TReadSessionSettings settings;
- for (const TString& topic : Settings.Topics) {
- settings.AppendTopics(topic);
- }
- settings.ConsumerName(Settings.ClientId);
- settings.MaxMemoryUsageBytes(Settings.MaxMemoryUsage);
- if (Settings.MaxTimeLagMs) {
- settings.MaxTimeLag(TDuration::MilliSeconds(Settings.MaxTimeLagMs));
- }
- if (Settings.ReadTimestampMs) {
- settings.StartingMessageTimestamp(TInstant::MilliSeconds(Settings.ReadTimestampMs));
- }
- if (Settings.PartitionGroups.size()) {
- Y_ENSURE(settings.Topics_.size() == 1);
- for (ui64 group : Settings.PartitionGroups) {
- settings.Topics_[0].AppendPartitionGroupIds(group);
- }
- }
- if (Settings.ReconnectOnFailure || Settings.ReadFromAllClusterSources) { // ReadFromAllClusterSources implies ReconnectOnFailure and MaxAttempts == inf.
- size_t maxRetries = Settings.MaxAttempts;
- if (Settings.ReadFromAllClusterSources) {
- // Compatibility.
- maxRetries = std::numeric_limits<size_t>::max();
- }
-
- if (Settings.UseV2RetryPolicyInCompatMode) {
- settings.RetryPolicy(
- NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(
- Settings.ReconnectionDelay,
- Settings.ReconnectionDelay,
- Settings.MaxReconnectionDelay,
- maxRetries,
- TDuration::Max(),
- 2.0,
- NYdb::NPersQueue::GetRetryErrorClassV2
- ));
- } else {
- settings.RetryPolicy(
- NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(
- Settings.ReconnectionDelay,
- Settings.ReconnectionDelay,
- Settings.MaxReconnectionDelay,
- maxRetries
- ));
- }
- } else {
- settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy());
- }
- if (Settings.ReadFromAllClusterSources) {
- settings.ReadAll();
- } else {
- if (TString cluster = FindClusterFromEndpoint(Settings.Server.Address)) {
- if (Settings.ReadMirroredPartitions) {
- settings.ReadMirrored(cluster);
- } else {
- settings.ReadOriginal({ cluster });
- }
- } else {
- settings.ReadAll();
- }
- }
- if (Settings.DisableCDS) {
- settings.DisableClusterDiscovery(true);
- }
-
- settings.EventHandlers_.HandlersExecutor(NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(PQLib->GetQueuePool().GetQueuePtr(this)));
- {
- auto weakThis = weak_from_this();
- settings.EventHandlers_.SessionClosedHandler(
- [weakThis](const NYdb::NPersQueue::TSessionClosedEvent& event) {
- if (auto sharedThis = weakThis.lock()) {
- const TString description = event.GetIssues().ToString();
- sharedThis->Destroy(description, ToErrorCode(event.GetStatus()));
- }
- }
- );
- }
-
- return settings;
-}
-
-NThreading::TFuture<TConsumerCreateResponse> TYdbSdkCompatibilityConsumer::Start(TInstant) noexcept {
- ReadSession = Client.CreateReadSession(YdbSdkSettings);
- SessionId = ReadSession->GetSessionId();
- DEBUG_LOG("Create read session", "", SessionId);
- SubscribeToNextEvent();
- TReadResponse resp;
- resp.MutableInit();
- return NThreading::MakeFuture<TConsumerCreateResponse>(TConsumerCreateResponse(std::move(resp)));
-}
-
-NThreading::TFuture<TError> TYdbSdkCompatibilityConsumer::IsDead() noexcept {
- return DeadPromise.GetFuture();
-}
-
-void TYdbSdkCompatibilityConsumer::Destroy(const TError& description) {
- if (DeadPromise.HasValue()) {
- return;
- }
-
- WARN_LOG("Destroying consumer: " << description, "", SessionId);
-
- while (!Requests.empty()) {
- NPersQueue::TReadResponse resp;
- *resp.MutableError() = description;
- Requests.front().SetValue(TConsumerMessage(std::move(resp)));
- Requests.pop();
- }
-
- if (ReadSession) {
- ReadSession->Close(TDuration::Zero());
- ReadSession = nullptr;
- }
-
- DeadPromise.SetValue(description);
-
- DestroyPQLibRef();
-}
-
-void TYdbSdkCompatibilityConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) {
- Destroy(MakeError(description, code));
-}
-
-void TYdbSdkCompatibilityConsumer::Cancel() {
- Destroy(GetCancelReason());
-}
-
-void TYdbSdkCompatibilityConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept {
- if (DeadPromise.HasValue()) {
- NPersQueue::TReadResponse resp;
- *resp.MutableError() = DeadPromise.GetFuture().GetValue();
- promise.SetValue(TConsumerMessage(std::move(resp)));
- return;
- }
- Requests.push(promise);
- AnswerToRequests();
- SubscribeToNextEvent();
-}
-
-void TYdbSdkCompatibilityConsumer::Commit(const TVector<ui64>& cookies) noexcept {
- NPersQueue::TReadResponse resp;
- for (ui64 cookie : cookies) {
- if (cookie >= NextCookie) {
- Destroy(TStringBuilder() << "Wrong cookie " << cookie, NErrorCode::WRONG_COOKIE);
- return;
- }
- auto offsetsIt = CookieToOffsets.find(cookie);
- if (offsetsIt != CookieToOffsets.end()) {
- offsetsIt->second.first.Commit();
- CookiesRequestedToCommit[offsetsIt->second.second].emplace(cookie);
- CookieToOffsets.erase(offsetsIt);
- } else {
- resp.MutableCommit()->AddCookie(cookie);
- }
- }
- if (resp.HasCommit()) {
- AddResponse(std::move(resp));
- }
-}
-
-void TYdbSdkCompatibilityConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept {
- Y_UNUSED(topic);
- Y_UNUSED(partition);
- auto partitionStreamIt = CurrentPartitionStreams.find(generation);
- if (partitionStreamIt != CurrentPartitionStreams.end()) {
- partitionStreamIt->second->RequestStatus();
- }
-}
-
-void TYdbSdkCompatibilityConsumer::AnswerToRequests() {
- TVector<NYdb::NPersQueue::TReadSessionEvent::TEvent> events;
- do {
- events = ReadSession->GetEvents();
- for (NYdb::NPersQueue::TReadSessionEvent::TEvent& event : events) {
- if (ReadSession) {
- std::visit([this](auto&& ev) { return HandleEvent(std::move(ev)); }, event);
- }
- }
- } while (!events.empty() && ReadSession);
-
- while (!Requests.empty() && !Responses.empty()) {
- Requests.front().SetValue(std::move(Responses.front()));
- Requests.pop();
- Responses.pop();
- }
-}
-
-void TYdbSdkCompatibilityConsumer::SubscribeToNextEvent() {
- if (!SubscribedToNextEvent && !Requests.empty() && ReadSession) {
- SubscribedToNextEvent = true;
- auto weakThis = weak_from_this();
- auto future = ReadSession->WaitEvent();
- Cerr << "SUBSCRIBING " << ReadSession->GetSessionId() << " future" << future.StateId()->Value() << "\n";
- PQLib->Subscribe(future, this, [weakThis](const NThreading::TFuture<void>&) {
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->OnReadSessionEvent();
- }
- });
- }
-}
-
-void TYdbSdkCompatibilityConsumer::OnReadSessionEvent() {
- if (DeadPromise.HasValue()) {
- return;
- }
-
- SubscribedToNextEvent = false;
- AnswerToRequests();
- SubscribeToNextEvent();
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event) {
- if (Settings.UseLockSession) {
- auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()];
- // Messages could contain holes in offset, but later commit ack will tell us right border.
- // So we can easily insert the whole interval with holes included.
- // It will be removed from set by specifying proper right border.
- offsetSet.InsertInterval(event.GetMessages().front().GetOffset(), event.GetMessages().back().GetOffset() + 1);
- }
- const ui64 cookie = NextCookie++;
- const auto& partitionStream = event.GetPartitionStream();
- auto& offsets = CookieToOffsets[cookie];
- offsets.first.Add(event);
- offsets.second = partitionStream->GetPartitionStreamId();
- PartitionStreamToCookies[partitionStream->GetPartitionStreamId()].emplace(cookie);
- NPersQueue::TReadResponse resp;
- auto& dataResp = *resp.MutableData();
- dataResp.SetCookie(cookie);
- auto& batchResp = *dataResp.AddMessageBatch();
- batchResp.SetTopic(MakeLegacyTopicName(partitionStream));
- batchResp.SetPartition(partitionStream->GetPartitionId());
- ui64 maxOffset = 0;
- for (auto&& msg : event.GetMessages()) {
- auto& msgResp = *batchResp.AddMessage();
- msgResp.SetOffset(msg.GetOffset());
- maxOffset = Max(maxOffset, msg.GetOffset());
- msgResp.SetData(msg.GetData());
- auto& metaResp = *msgResp.MutableMeta();
- metaResp.SetSourceId(msg.GetMessageGroupId());
- metaResp.SetSeqNo(msg.GetSeqNo());
- metaResp.SetCreateTimeMs(msg.GetCreateTime().MilliSeconds());
- metaResp.SetWriteTimeMs(msg.GetWriteTime().MilliSeconds());
- metaResp.SetIp(msg.GetIp());
- for (auto&& [k, v] : msg.GetMeta()->Fields) {
- auto& kvResp = *metaResp.MutableExtraFields()->AddItems();
- kvResp.SetKey(k);
- kvResp.SetValue(v);
- }
- }
- MaxOffsetToCookie[std::make_pair(partitionStream->GetPartitionStreamId(), maxOffset)] = cookie;
- AddResponse(std::move(resp));
-}
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event) {
- const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId();
- if (Settings.UseLockSession) {
- auto& offsetSet = PartitionStreamToUncommittedOffsets[partitionStreamId];
- if (offsetSet.EraseInterval(0, event.GetCommittedOffset())) { // Remove some offsets.
- if (offsetSet.Empty()) { // No offsets left.
- auto unconfirmedDestroyIt = UnconfirmedDestroys.find(partitionStreamId);
- if (unconfirmedDestroyIt != UnconfirmedDestroys.end()) {
- // Confirm and forget about this partition stream.
- unconfirmedDestroyIt->second.Confirm();
- UnconfirmedDestroys.erase(unconfirmedDestroyIt);
- PartitionStreamToUncommittedOffsets.erase(partitionStreamId);
- }
- }
- }
- }
- const auto offsetPair = std::make_pair(partitionStreamId, event.GetCommittedOffset() - 1);
- auto cookieIt = MaxOffsetToCookie.lower_bound(offsetPair);
- std::vector<ui64> cookies;
- auto end = cookieIt;
- auto begin = end;
- if (cookieIt != MaxOffsetToCookie.end() && cookieIt->first == offsetPair) {
- cookies.push_back(cookieIt->second);
- ++end;
- }
- while (cookieIt != MaxOffsetToCookie.begin()) {
- --cookieIt;
- if (cookieIt->first.first == partitionStreamId) {
- cookies.push_back(cookieIt->second);
- begin = cookieIt;
- } else {
- break;
- }
- }
- if (begin != end) {
- MaxOffsetToCookie.erase(begin, end);
- }
-
- NPersQueue::TReadResponse resp;
- auto& respCommit = *resp.MutableCommit();
- auto& partitionStreamCookies = PartitionStreamToCookies[partitionStreamId];
- auto& committedCookies = CookiesRequestedToCommit[partitionStreamId];
- for (auto committedCookieIt = cookies.rbegin(), committedCookieEnd = cookies.rend(); committedCookieIt != committedCookieEnd; ++committedCookieIt) {
- const ui64 cookie = *committedCookieIt;
- respCommit.AddCookie(cookie);
- partitionStreamCookies.erase(cookie);
- committedCookies.erase(cookie);
- }
- AddResponse(std::move(resp));
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event) {
- if (Settings.UseLockSession) {
- Y_VERIFY(PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()].Empty());
-
- NPersQueue::TReadResponse resp;
- auto& lockResp = *resp.MutableLock();
- const auto& partitionStream = event.GetPartitionStream();
- lockResp.SetTopic(MakeLegacyTopicName(partitionStream));
- lockResp.SetPartition(partitionStream->GetPartitionId());
- lockResp.SetReadOffset(event.GetCommittedOffset());
- lockResp.SetEndOffset(event.GetEndOffset());
- lockResp.SetGeneration(partitionStream->GetPartitionStreamId());
- NThreading::TPromise<TLockInfo> confirmPromise = NThreading::NewPromise<TLockInfo>();
- confirmPromise.GetFuture().Subscribe([event = std::move(event)](const NThreading::TFuture<TLockInfo>& infoFuture) mutable {
- const TLockInfo& info = infoFuture.GetValue();
- event.Confirm(info.ReadOffset, info.CommitOffset);
- }); // It doesn't matter in what thread this callback will be called.
- AddResponse(std::move(resp), std::move(confirmPromise));
- } else {
- event.Confirm();
- }
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event) {
- const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId();
- CurrentPartitionStreams[partitionStreamId] = event.GetPartitionStream();
- if (Settings.UseLockSession) {
- const auto partitionStream = event.GetPartitionStream();
-
- Y_VERIFY(UnconfirmedDestroys.find(partitionStreamId) == UnconfirmedDestroys.end());
- if (PartitionStreamToUncommittedOffsets[partitionStreamId].Empty() || Settings.BalanceRightNow) {
- PartitionStreamToUncommittedOffsets.erase(partitionStreamId);
- event.Confirm();
- } else {
- UnconfirmedDestroys.emplace(partitionStreamId, std::move(event));
- }
-
- NPersQueue::TReadResponse resp;
- auto& releaseResp = *resp.MutableRelease();
- releaseResp.SetTopic(MakeLegacyTopicName(partitionStream));
- releaseResp.SetPartition(partitionStream->GetPartitionId());
- releaseResp.SetCanCommit(true);
- releaseResp.SetGeneration(partitionStream->GetPartitionStreamId());
- AddResponse(std::move(resp));
- } else {
- event.Confirm();
- }
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event) {
- NPersQueue::TReadResponse resp;
- auto& statusResp = *resp.MutablePartitionStatus();
- const auto& partitionStream = event.GetPartitionStream();
- statusResp.SetGeneration(partitionStream->GetPartitionStreamId());
- statusResp.SetTopic(MakeLegacyTopicName(partitionStream));
- statusResp.SetPartition(partitionStream->GetPartitionId());
- statusResp.SetCommittedOffset(event.GetCommittedOffset());
- statusResp.SetEndOffset(event.GetEndOffset());
- statusResp.SetWriteWatermarkMs(event.GetWriteWatermark().MilliSeconds());
- AddResponse(std::move(resp));
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event) {
- const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId();
- CurrentPartitionStreams.erase(partitionStreamId);
-
- {
- NPersQueue::TReadResponse resp;
- auto& respCommit = *resp.MutableCommit();
- for (ui64 cookie : CookiesRequestedToCommit[partitionStreamId]) {
- respCommit.AddCookie(cookie);
- }
- if (resp.GetCommit().CookieSize()) {
- AddResponse(std::move(resp));
- }
- CookiesRequestedToCommit.erase(partitionStreamId);
- }
-
- {
- for (ui64 cookie : PartitionStreamToCookies[partitionStreamId]) {
- CookieToOffsets.erase(cookie);
- }
- PartitionStreamToCookies.erase(partitionStreamId);
- }
-
- {
- auto begin = MaxOffsetToCookie.lower_bound(std::make_pair(partitionStreamId, 0)); // The first available offset.
- if (begin != MaxOffsetToCookie.end() && begin->first.first == partitionStreamId) {
- auto end = begin;
- while (end != MaxOffsetToCookie.end() && end->first.first == partitionStreamId) {
- ++end;
- }
- MaxOffsetToCookie.erase(begin, end);
- }
- }
-
- if (Settings.UseLockSession) {
- PartitionStreamToUncommittedOffsets.erase(partitionStreamId);
- UnconfirmedDestroys.erase(partitionStreamId);
-
- if (event.GetReason() != NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::DestroyConfirmedByUser) {
- NPersQueue::TReadResponse resp;
- auto& releaseResp = *resp.MutableRelease();
- const auto& partitionStream = event.GetPartitionStream();
- releaseResp.SetTopic(MakeLegacyTopicName(partitionStream));
- releaseResp.SetPartition(partitionStream->GetPartitionId());
- releaseResp.SetCanCommit(false);
- releaseResp.SetGeneration(partitionStream->GetPartitionStreamId());
- AddResponse(std::move(resp));
- }
- }
-}
-
-void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event) {
- Destroy(event.GetIssues().ToString(), ToErrorCode(event.GetStatus()));
-}
-
-void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead) {
- Responses.emplace(std::move(response), std::move(readyToRead));
-}
-
-void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response) {
- Responses.emplace(std::move(response));
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h
deleted file mode 100644
index 0ff6756686..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#pragma once
-#include "iconsumer_p.h"
-
-#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h>
-
-#include <library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h>
-
-#include <queue>
-
-namespace NPersQueue {
-
-class TYdbSdkCompatibilityConsumer : public IConsumerImpl,
- public std::enable_shared_from_this<TYdbSdkCompatibilityConsumer>
-{
-public:
- TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings,
- std::shared_ptr<void> destroyEventRef,
- TIntrusivePtr<TPQLibPrivate> pqLib,
- TIntrusivePtr<ILogger> logger,
- NYdb::NPersQueue::TPersQueueClient& client);
- ~TYdbSdkCompatibilityConsumer();
-
- void Init() override;
-
- NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override;
- NThreading::TFuture<TError> IsDead() noexcept override;
- void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override;
- void Commit(const TVector<ui64>& cookies) noexcept override;
- void Cancel() override;
- void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override;
-
-private:
- class TReadSessionEventVisitor;
-
- NYdb::NPersQueue::TReadSessionSettings MakeReadSessionSettings();
- void SubscribeToNextEvent();
- void OnReadSessionEvent();
- void AnswerToRequests();
- void Destroy(const TError& description);
- void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR);
-
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event);
- void HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event);
-
- void AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead);
- void AddResponse(TReadResponse&& response);
-
-private:
- const TConsumerSettings Settings;
- NYdb::NPersQueue::TReadSessionSettings YdbSdkSettings;
- TIntrusivePtr<ILogger> Logger;
- TString SessionId;
- NYdb::NPersQueue::TPersQueueClient& Client;
- std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession;
- NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>();
- std::queue<NThreading::TPromise<TConsumerMessage>> Requests;
- std::queue<TConsumerMessage> Responses;
- bool SubscribedToNextEvent = false;
- THashMap<ui64, std::pair<NYdb::NPersQueue::TDeferredCommit, ui64>> CookieToOffsets; // Cookie -> { commit, partition stream id }.
- TMap<std::pair<ui64, ui64>, ui64> MaxOffsetToCookie; // { partition stream id, max offset of cookie } -> cookie.
- THashMap<ui64, THashSet<ui64>> CookiesRequestedToCommit; // Partition stream id -> cookies that user requested to commit.
- THashMap<ui64, THashSet<ui64>> PartitionStreamToCookies; // Partition stream id -> cookies.
- ui64 NextCookie = 1;
- // Commits for graceful release partition after commit.
- THashMap<ui64, TDisjointIntervalTree<ui64>> PartitionStreamToUncommittedOffsets; // Partition stream id -> set of offsets.
- THashMap<ui64, NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent> UnconfirmedDestroys; // Partition stream id -> destroy events.
- THashMap<ui64, NYdb::NPersQueue::TPartitionStream::TPtr> CurrentPartitionStreams;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h
deleted file mode 100644
index 8b429053fd..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h
+++ /dev/null
@@ -1,61 +0,0 @@
-#pragma once
-
-#include "types.h"
-#include "responses.h"
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/map.h>
-#include <util/generic/vector.h>
-
-namespace NPersQueue {
-
-struct TPartition {
- TString Topic;
- ui32 PartitionId = 0;
-
- bool operator <(const TPartition& other) const {
- return std::tie(Topic, PartitionId) < std::tie(other.Topic, other.PartitionId);
- }
-
- explicit TPartition(const TString& topic, const ui32 partitionId)
- : Topic(topic)
- , PartitionId(partitionId)
- {}
-};
-
-struct TProcessedMessage {
- TString Topic;
- TString Data;
- TString SourceIdPrefix;
- ui32 Group = 0;
-};
-
-struct TProcessedData {
- TVector<TProcessedMessage> Messages; // maybe grouped by topic?
-};
-
-struct TOriginMessage {
- TReadResponse::TData::TMessage Message;
- NThreading::TPromise<TProcessedData> Processed;
-};
-
-struct TOriginData {
- TMap<TPartition, TVector<TOriginMessage>> Messages;
-};
-
-
-class IProcessor {
-public:
- virtual ~IProcessor() = default;
-
- // Returns data and promise for each message.
- // Client MUST process each message individually in a deterministic way.
- // Each original message CAN be transformed into several messages and written to several topics/sourceids.
- // But for any given original message all pairs (topic, sourceIdPrefix) of resulting messages MUST be unique.
- // Client MUST finish processing by signaling corresponding promise.
- // Otherwise, exactly-once guarantees cannot be hold.
- virtual NThreading::TFuture<TOriginData> GetNextData() noexcept = 0;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h
deleted file mode 100644
index cbb221e9ee..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include "types.h"
-#include "responses.h"
-#include <library/cpp/threading/future/future.h>
-
-namespace NPersQueue {
-
-// IProducer is threadsafe.
-// If one creates several producers with same SourceId in same topic - first Producer (from point of server view) will die.
-// There could be only one producer with concrete SourceId in concrete topic at once.
-
-class IProducer {
-public:
- // Start producer.
- // Producer can be used after its start will be finished.
- virtual NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0;
-
- NThreading::TFuture<TProducerCreateResponse> Start(TDuration timeout) noexcept {
- return Start(TInstant::Now() + timeout);
- }
-
- // Add write request to queue.
- virtual NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept = 0;
-
- // Add write request to queue without specifying seqNo. So, without deduplication (at least once guarantee).
- virtual NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept = 0;
-
- // Get future that is signalled when producer is dead.
- virtual NThreading::TFuture<TError> IsDead() noexcept = 0;
-
- virtual ~IProducer() = default;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h
deleted file mode 100644
index bb3320d795..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-
-namespace NPersQueue {
-
-class ILogger : public TAtomicRefCount<ILogger> {
-public:
- virtual ~ILogger() = default;
- //level = syslog level
- virtual void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) = 0;
- virtual bool IsEnabled(int level) const = 0;
-};
-
-class TCerrLogger : public ILogger {
-public:
- explicit TCerrLogger(int level)
- : Level(level)
- {
- }
-
- ~TCerrLogger() override = default;
-
- void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override;
-
- bool IsEnabled(int level) const override
- {
- return level <= Level;
- }
-
- int GetLevel() const {
- return Level;
- }
-
-private:
- static TStringBuf LevelToString(int level);
-
-private:
- int Level;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h
deleted file mode 100644
index 1b1f7646c8..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-
-#include "types.h"
-#include "iproducer.h"
-#include "iconsumer.h"
-#include "iprocessor.h"
-#include "responses.h"
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/noncopyable.h>
-#include <util/generic/ptr.h>
-
-namespace NPersQueue {
-
-class TPQLibPrivate;
-
-class TPQLib : public TNonCopyable {
-public:
- explicit TPQLib(const TPQLibSettings& settings = TPQLibSettings());
- ~TPQLib();
-
- // Deprecated flag means that PQLib doen't wait its objects during destruction.
- // This behaviour leads to potential thread leakage and thread sanitizer errors.
- // It is recommended to specify deprecated = false. In that case PQLib will cancel
- // all objects and correctly wait for its threads in destructor. But, unfortunately,
- // we can't migrate all current clients to this default behaviour automatically,
- // because this can break existing programs unpredictably.
-
- // Producers creation
- THolder<IProducer> CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false);
- THolder<IProducer> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false);
-
- // Consumers creation
- THolder<IConsumer> CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false);
-
- // Processors creation
- THolder<IProcessor> CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false);
-
- void SetLogger(TIntrusivePtr<ILogger> logger);
-
- TString GetUserAgent() const;
- void SetUserAgent(const TString& userAgent);
-
-private:
- TIntrusivePtr<TPQLibPrivate> Impl;
- TAtomic Alive = 1; // Debug check for valid PQLib usage.
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h
deleted file mode 100644
index de5a801172..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h
+++ /dev/null
@@ -1,106 +0,0 @@
-#pragma once
-
-#include "types.h"
-
-#include <library/cpp/threading/future/future.h>
-#include <util/generic/string.h>
-
-namespace NPersQueue {
-
-struct TProducerCreateResponse {
- TProducerCreateResponse(TWriteResponse&& response)
- : Response(std::move(response))
- {
- }
-
- TWriteResponse Response;
-};
-
-struct TProducerCommitResponse {
- TProducerCommitResponse(TProducerSeqNo seqNo, TData data, TWriteResponse&& response)
- : Response(std::move(response))
- , Data(std::move(data))
- , SeqNo(seqNo)
- {
- }
-
- TWriteResponse Response;
- TData Data;
- TProducerSeqNo SeqNo;
-};
-
-
-struct TConsumerCreateResponse {
- TConsumerCreateResponse(TReadResponse&& response)
- : Response(std::move(response))
- {
- }
-
- //will contain Error or Init
- TReadResponse Response;
-};
-
-enum EMessageType {
- EMT_LOCK,
- EMT_RELEASE,
- EMT_DATA,
- EMT_ERROR,
- EMT_STATUS,
- EMT_COMMIT
-};
-
-struct TLockInfo {
- ui64 ReadOffset = 0;
- ui64 CommitOffset = 0;
- bool VerifyReadOffset = false;
-
- TLockInfo() = default;
-
- // compatibility with msvc2015
- TLockInfo(ui64 readOffset, ui64 commitOffset, bool verifyReadOffset)
- : ReadOffset(readOffset)
- , CommitOffset(commitOffset)
- , VerifyReadOffset(verifyReadOffset)
- {}
-};
-
-static EMessageType GetType(const TReadResponse& response) {
- if (response.HasData()) return EMT_DATA;
- if (response.HasError()) return EMT_ERROR;
- if (response.HasCommit()) return EMT_COMMIT;
- if (response.HasPartitionStatus()) return EMT_STATUS;
- if (response.HasRelease()) return EMT_RELEASE;
- if (response.HasLock()) return EMT_LOCK;
-
- // for no warn return anything.
- return EMT_LOCK;
-}
-
-
-
-struct TConsumerMessage {
- EMessageType Type;
- //for DATA/ERROR/LOCK/RELEASE/COMMIT/STATUS:
- //will contain Error, Data, Lock or Release and be consistent with Type
- TReadResponse Response;
-
- //for LOCK only:
- mutable NThreading::TPromise<TLockInfo> ReadyToRead;
-
- TConsumerMessage(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead)
- : Type(EMT_LOCK)
- , Response(std::move(response))
- , ReadyToRead(std::move(readyToRead))
- {
- Y_VERIFY(GetType(Response) == EMT_LOCK);
- }
-
-
- explicit TConsumerMessage(TReadResponse&& response)
- : Type(GetType(response))
- , Response(std::move(response))
- {
- }
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp
deleted file mode 100644
index b30115e294..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp
+++ /dev/null
@@ -1,236 +0,0 @@
-#include <util/datetime/base.h>
-#include <util/stream/file.h>
-#include <util/string/join.h>
-#include <util/string/vector.h>
-
-#include <google/protobuf/message.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-using namespace NPersQueue;
-
-
-int main() {
-
- // [BEGIN create pqlib]
- // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib.
- TIntrusivePtr<ILogger> logger = new TCerrLogger(7);
-
- // Step 2. Create settings of main TPQLib object.
- TPQLibSettings pqLibSettings;
-
- pqLibSettings.DefaultLogger = logger;
-
- // Step 3. Create TPQLib object.
- TPQLib pq(pqLibSettings);
- // [END create pqlib]
-
- // [BEGIN create consumer]
- // Step 4. Parameters used in sample.
-
- // List of topics to read.
- TVector<TString> topics = {"topic_path"};
-
- // Path of consumer that will read topics.
- TString consumerPath = "consumer_path";
-
- // OAuth token.
- TString oauthToken = "oauth_token";
-
- // IAM JWT key
- TString jwtKey = "<jwt json key>";
-
- // TVM parameters.
- TString tvmSecret = "tvm_secret";
- ui32 tvmClientId = 200000;
- ui32 tvmDstId = 2001147; // logbroker main prestable
- tvmDstId = 2001059; // logbroker main production
-
- // Logbroker endpoint.
- TString endpoint = "man.logbroker.yandex.net";
-
- // Simple single read.
-
- // Step 5. Create settings of consumer object.
- TConsumerSettings settings;
- // Fill consumer settings. See types.h for more information.
- // Settings without defaults, must be set always.
- settings.Server.Address = endpoint;
- {
- // Working with logbroker in Yandex.Cloud:
- // Fill database for Yandex.Cloud Logbroker.
- settings.Server.Database = "/Root";
- // Enable TLS.
- settings.Server.EnableSecureConnection("");
- }
- settings.Topics = topics;
- settings.ClientId = consumerPath;
-
- // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers.
- // Create OAuth provider from OAuth ticket.
- settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken);
- // Or create TVM provider from TVM settings.
- settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger);
- // Or create IAM provider from jwt-key
- settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger);
-
- // Step 6. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB.
- auto consumer = pq.CreateConsumer(settings, logger);
-
- // Step 7. Start consuming data.
- auto future = consumer->Start();
-
- // Step 9. Wait initialization and check it result.
- future.Wait();
-
- if (future.GetValue().Response.HasError()) {
- return 1;
- }
- // [END create consumer]
-
- // [BEGIN read messages]
- // Step 10. Simple single read.
- // Step 10.1. Request message from consumer.
- auto msg = consumer->GetNextMessage();
-
- // Step 10.2. Wait for message.
- msg.Wait();
-
- auto value = msg.GetValue();
-
- // Step 10.3. Check result.
- switch (value.Type) {
- // Step 10.3.1. Result is error - consumer is in incorrect state now. You can log error and recreate everything starting from step 5.
- case EMT_ERROR:
- return 1;
- // Step 10.3.2. Read data.
- case EMT_DATA: {
- // Process received messages. Inside will be several batches.
- for (const auto& t : value.Response.data().message_batch()) {
- const TString topic = t.topic();
- const ui32 partition = t.partition();
- Y_UNUSED(topic);
- Y_UNUSED(partition);
- // There are several messages from partition of topic inside one batch.
- for (const auto& m : t.message()) {
- const ui64 offset = m.offset();
- const TString& data = m.data();
- // Each message has offset in partition, payload data (decompressed or as is) and metadata.
- Y_UNUSED(offset);
- Y_UNUSED(data);
- }
- }
-
- // Step 10.3.3. Each processed data batch from step 10.3.2 must be committed once all data inside is processed.
-
- // This must be done using cookie from data batch.
- auto cookie = msg.GetValue().Response.GetData().GetCookie();
- // Commit cookie['s]. Acknowledge from server will be received inside one of next messages from consumer later.
- consumer->Commit({cookie});
- break;
- }
- // Step 10.3.4. Commit acknowledge. Can be used for logging and debugging purposes. (Can't be first message, only possible if one makes several
- // consecutive reads.
- case EMT_COMMIT:
- break;
- // Handle other cases - only for compiling, there could not be any.
- default:
- break;
-
- }
- // [END read messages]
-
- // [BEGIN advanced read messages]
-
- // Point that consumer will perform advanced reads.
- settings.UseLockSession = true;
-
- {
- // Step 11. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB.
- auto consumer = pq.CreateConsumer(settings, logger);
-
- // Step 12. Start consuming data.
- auto future = consumer->Start();
-
- // Step 14. Wait initialization and check it result.
- future.Wait();
-
- if (future.GetValue().Response.HasError()) {
- return 1;
- }
-
- // Step 14. Event loop.
- while (true) {
- // Step 14.1. Request message from consumer.
- auto msg = consumer->GetNextMessage();
-
- // Step 14.2. Wait for message.
- msg.Wait();
-
- const auto& value = msg.GetValue();
-
- // Step 14.3. Check result.
- switch (value.Type) {
- // Step 14.3.1. Result is error: consumer is in incorrect state now. You can log error and recreate everything starting from step 11.
- case EMT_ERROR:
- return 1;
- // Step 14.3.2. Lock request from server.
- case EMT_LOCK: {
- const TString& topic = value.Response.lock().topic();
- const ui32 partition = value.Response.lock().partition();
- // Server is ready to send data from corresponding partition of topic. Topic is legacy topic name = rt3.<cluster>--<converted topic path>
- Y_UNUSED(topic);
- Y_UNUSED(partition);
- // One must react on this event with setting promise ReadyToRead right now or later. Only after this client will start receiving data from this partition.
- msg.GetValue().ReadyToRead.SetValue(TLockInfo{0 /*readOffset*/, 0 /*commitOffset*/, false /*verifyReadOffset*/});
- break;
- }
- // Step 14.3.3. Release request from server.
- case EMT_RELEASE: {
- const TString& topic = value.Response.release().topic();
- const ui32 partition = value.Response.release().partition();
- Y_UNUSED(topic);
- Y_UNUSED(partition);
- // // Server will not send any data from partition of topic until next Lock request is confirmed by client.
- break;
- }
- // Step 14.3.4. Read data.
- case EMT_DATA: {
- // Process received messages. Inside will be several batches. The same as in the simple read.
- for (const auto& t : value.Response.data().message_batch()) {
- const TString topic = t.topic();
- const ui32 partition = t.partition();
- Y_UNUSED(topic);
- Y_UNUSED(partition);
- // There are several messages from partition of topic inside one batch.
- for (const auto& m : t.message()) {
- const ui64 offset = m.offset();
- const TString& data = m.data();
- // Each message has offset in partition, payload data (decompressed or as is) and metadata.
- Y_UNUSED(offset);
- Y_UNUSED(data);
- }
- }
-
- // Step 14.3.5. Each processed data batch from step 14.3.4 should be committed after all data inside is processed.
-
- // This must be done using cookie from data batch.
- auto cookie = msg.GetValue().Response.GetData().GetCookie();
- // Commit cookie['s]. Acknowledge from server will be received inside message from consumer later.
- consumer->Commit({cookie});
- break;
- }
- // Step 14.3.6. Commit acknowledge. Can be used for logging and debugging purposes.
- case EMT_STATUS:
- case EMT_COMMIT:
- break;
- } // switch()
- } // while()
- }
-
- // [END advanced read messages]
-
- // Step 15. End of working. Nothing special needs to be done - desctuction of objects only.
-
- return 0;
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp
deleted file mode 100644
index c91838892b..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp
+++ /dev/null
@@ -1,174 +0,0 @@
-#include <queue>
-
-#include <util/datetime/base.h>
-#include <util/stream/file.h>
-#include <util/string/join.h>
-#include <util/string/vector.h>
-
-#include <google/protobuf/message.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-using namespace NPersQueue;
-
-
-int main() {
-
-
- // [BEGIN create pqlib]
- // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib.
- TIntrusivePtr<ILogger> logger = new TCerrLogger(7);
-
- // Step 2. Create settings of main TPQLib object.
- TPQLibSettings pqLibSettings;
-
- // Number of threads for processing gRPC events.
- pqLibSettings.ThreadsCount = 1;
- // Number of threads for compression/decomression actions.
- pqLibSettings.CompressionPoolThreads = 3;
- // Default logger. Used inside consumers and producers if no logger provided.
- pqLibSettings.DefaultLogger = logger;
-
- // Step 3. Create TPQLib object.
- TPQLib pq(pqLibSettings);
- // [END create pqlib]
-
- // [BEGIN create producer]
- // Step 4. Initialize producer parameters
-
- // Topics to read.
- TString topic = "topic_path";
- // Partitions group. 0 means any.
- ui32 group = 0;
-
- // Source identifier of messages group.
- TString sourceId = "source_id";
-
- // OAuth token.
- TString oauthToken = "oauth_token";
-
- // IAM JWT key
- TString jwtKey = "<jwt json key>";
-
- // TVM parameters.
- TString tvmSecret = "tvm_secret";
- ui32 tvmClientId = 200000;
- ui32 tvmDstId = 2001147; // logbroker main prestable
- tvmDstId = 2001059; // logbroker main production
-
- // Logbroker endpoint.
- TString endpoint = "logbroker.yandex.net";
-
- // Step 5. Put producer parameters into settings.
- TProducerSettings settings;
- // Logbroker endpoint.
- settings.Server.Address = endpoint;
- {
- // Working with logbroker in Yandex.Cloud:
- // Fill database for Yandex.Cloud Logbroker.
- settings.Server.Database = "/Root";
- // Enable TLS.
- settings.Server.EnableSecureConnection("");
- }
- // Topic path.
- settings.Topic = topic;
- // SourceId of messages group.
- settings.SourceId = sourceId;
- // Partition group to write to.
- settings.PartitionGroup = group;
- // ReconnectOnFailure will gurantee that producer will retry writes in case of failre.
- settings.ReconnectOnFailure = true; //retries on server errors
- // Codec describes how to compress data.
- settings.Codec = NPersQueueCommon::ECodec::GZIP;
-
- // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers.
- // Create OAuth provider from OAuth ticket.
- settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken);
- // Or create TVM provider from TVM settings.
- settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger);
- // Or create IAM provider from jwt-key
- settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger);
-
- // Step 6. Create producer object. PQLib object must be valid during all lifetime of producer, otherwise writing to producer will fail but this is not UB.
- auto producer = pq.CreateProducer(settings, logger);
-
- // Step 7. Start producer.
- auto future = producer->Start();
-
- // Step 8. Wait initialization and check it result.
- future.Wait();
-
- if (future.GetValue().Response.HasError()) {
- return 1;
- }
- // [END create producer]
-
- // [BEGIN write message]
-
- // Step 9. Single write sample.
- // Step 9.1. Form message.
- // Creation timestamp.
- TInstant timestamp = TInstant::Now();
- // Message payload.
- TString payload = "abacaba";
-
- TData data{payload, timestamp};
- // Corresponding messages's sequence number.
- ui64 seqNo = 1;
-
- // Step 9.2. Write message. Until result future is signalled - message is stored inside PQLib memory. Future will be signalled with message payload and result of writing.
- auto result = producer->Write(seqNo, data);
-
- // Step 9.3 Check writing result.
- // Check all results.
- result.Wait();
- Y_VERIFY(!result.GetValue().Response.HasError());
-
- // Only when future is destroyed no memory is used by message.
-
- // [END write message]
-
- // [BEGIN write messages with inflight]
-
- // Step 10. Writing loop with in-flight.
- // Max in-flight border, parameter.
- ui32 maxInflyCount = 10;
- // Queue for messages in-flight.
- std::queue<NThreading::TFuture<TProducerCommitResponse>> v;
-
- while(true) {
- // Creation timestamp.
- TInstant timestamp = TInstant::Now();
- // Message payload.
- TString payload = "abacaba";
- TData data{payload, timestamp};
-
- // SeqNo must increase for consecutive messages.
- ++seqNo;
-
- // Step 10.1. Write message.
- auto result = producer->Write(seqNo, data);
- // Step 10.2. Put future inside queue.
- v.push(result);
- // Step 10.3. If in-flight size is too big.
- if (v.size() > maxInflyCount) {
- // Step 10.4. Wait for first message in queue processing result, check it.
- v.front().Wait();
- Y_VERIFY(!v.front().GetValue().Response.HasError());
- // Step 10.5. Drop first processed message. Only after memory of this message will be freed.
- v.pop();
- }
- }
-
- // Step 10.6. After processing loop one must wait for all not yet processed messages.
-
- while (!v.empty()) {
- v.front().Wait();
- Y_VERIFY(!v.front().GetValue().Response.HasError());
- v.pop();
- }
-
- // [END write messages with inflight]
-
- return 0;
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h
deleted file mode 100644
index 37c4423d6f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h
+++ /dev/null
@@ -1,402 +0,0 @@
-#pragma once
-#include "credentials_provider.h"
-#include "logger.h"
-#include <kikimr/yndx/api/protos/persqueue.pb.h>
-
-
-#include <library/cpp/logger/priority.h>
-
-#include <util/generic/string.h>
-#include <util/generic/buffer.h>
-#include <util/generic/maybe.h>
-#include <util/generic/vector.h>
-#include <util/generic/hash.h>
-#include <util/datetime/base.h>
-#include <util/stream/output.h>
-#include <util/string/builder.h>
-
-#include <grpc++/channel.h>
-
-#include <limits>
-
-namespace NPersQueue {
-using ECodec = NPersQueueCommon::ECodec;
-using TCredentials = NPersQueueCommon::TCredentials;
-using TError = NPersQueueCommon::TError;
-
-using TProducerSeqNo = ui64; // file offset or something with equivalent meaning.
-using TConsumerOffset = i64; // partition offset. Have special value -1. All other values are >= 0.
-using TCredProviderPtr = std::shared_ptr<ICredentialsProvider>;
-
-constexpr ui32 DEFAULT_MEMORY_USAGE = 100 << 20; //100mb
-constexpr ui32 DEFAULT_READS_INFLY = 10;
-constexpr ui32 DEFAULT_BATCH_SIZE = 1 << 20; //1mb
-constexpr TDuration DEFAULT_CHANNEL_CREATION_TIMEOUT = TDuration::Seconds(10);
-constexpr TDuration DEFAULT_RECONNECTION_DELAY = TDuration::MilliSeconds(10);
-constexpr TDuration DEFAULT_MAX_RECONNECTION_DELAY = TDuration::Seconds(1);
-constexpr TDuration DEFAULT_START_SESSION_TIMEOUT = TDuration::Seconds(30);
-constexpr TDuration DEFAULT_CHECK_TOKEN_PERIOD = TDuration::Hours(1);
-inline const THashMap<ECodec, TString> CodecIdByType{{ECodec::DEFAULT, TString(1, '\0')}, {ECodec::RAW, TString(1, '\0')}, {ECodec::GZIP, TString(1, '\1')}, {ECodec::LZOP, TString(1, '\2')}, {ECodec::ZSTD, TString(1, '\3')}};
-
-struct TPQLibSettings {
- size_t ThreadsCount = 1;
- size_t CompressionPoolThreads = 2;
- size_t GRpcThreads = 1;
- TDuration ChannelCreationTimeout = DEFAULT_CHANNEL_CREATION_TIMEOUT;
- TIntrusivePtr<ILogger> DefaultLogger = {};
-
- bool EnableGRpcV1 = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED");
-};
-
-enum class EClusterDiscoveryUsageMode {
- Auto,
- Use,
- DontUse,
-};
-
-// destination server
-struct TServerSetting {
- // Endpoint of logbroker cluster to connect.
- // Possible variants:
- //
- // 1. lbkx cluster.
- // Set address to lbkx.logbroker.yandex.net
- //
- // 2. logbroker federation.
- // If you want to write/read to/from concrete cluster
- // (with ReadMirroredPartitions or not, but without ReadFromAllClusterSources option),
- // use the following addresses:
- // iva.logbroker.yandex.net
- // man.logbroker.yandex.net
- // myt.logbroker.yandex.net
- // sas.logbroker.yandex.net
- // vla.logbroker.yandex.net
- //
- // If you create consumer with ReadFromAllClusterSources option,
- // use logbroker.yandex.net
- TString Address;
-
- ui16 Port = 2135;
- TString Database = GetRootDatabase();
- EClusterDiscoveryUsageMode UseLogbrokerCDS = EClusterDiscoveryUsageMode::Auto;
- bool UseSecureConnection = false;
- TString CaCert = TString();
-
- TServerSetting(TString address, ui16 port = 2135)
- : Address(std::move(address))
- , Port(port)
- {
- Y_VERIFY(!Address.empty());
- Y_VERIFY(Port > 0);
- }
-
- TServerSetting() = default;
- TServerSetting(const TServerSetting&) = default;
- TServerSetting(TServerSetting&&) = default;
-
- TServerSetting& operator=(const TServerSetting&) = default;
- TServerSetting& operator=(TServerSetting&&) = default;
-
- TString GetFullAddressString() const {
- return TStringBuilder() << Address << ':' << Port;
- }
- void EnableSecureConnection(const TString& caCert) {
- UseSecureConnection = true;
- CaCert = caCert;
- }
- void DisableSecureConnection() {
- UseSecureConnection = false;
- CaCert.clear();
- }
-
- const TString& GetRootDatabase();
-
- bool operator < (const TServerSetting& rhs) const {
- return std::tie(Address, Port, Database, UseSecureConnection) < std::tie(rhs.Address, rhs.Port, rhs.Database, rhs.UseSecureConnection);
- }
-};
-
-//
-// consumer/producer settings
-//
-
-struct TProducerSettings {
- TServerSetting Server;
- std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider();
-
- TString Topic;
- TString SourceId;
-
- ui32 PartitionGroup = 0;
- ECodec Codec = ECodec::GZIP;
- int Quality = -1;
- THashMap<TString, TString> ExtraAttrs;
-
- // Should we reconnect when session breaks?
- bool ReconnectOnFailure = false;
-
- //
- // Setting for reconnection
- //
-
- // In case of lost connection producer will try
- // to reconnect and resend data <= MaxAttempts times.
- // Then producer will be dead.
- unsigned MaxAttempts = std::numeric_limits<unsigned>::max();
-
- // Time delay before next producer reconnection attempt.
- // The second attempt will be in 2 * ReconnectionDelay,
- // the third - in 4 * ReconnectionDelay, etc. Maximum value
- // of delay is MaxReconnectionDelay.
- TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY;
- TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY;
-
- // Timeout for session initialization operation.
- TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT;
- // Producer will check token from 'ICredentialsProvider' with this period and if changed - send to server as soon as possible.
- // Available only with gRPC data plane API v1.
- TDuration CheckTokenPeriod = DEFAULT_CHECK_TOKEN_PERIOD;
-
- // Preferred cluster to write.
- // This setting is used only in case of cluster discovery is used.
- // Case insensitive.
- // If PreferredCluster is not available,
- // producer will connect to other cluster.
- TString PreferredCluster;
-
- bool DisableCDS = false; // only for grpcv1 mode
-};
-
-struct TMultiClusterProducerSettings {
- struct TServerWeight {
- // Setting of producer connected to concrete DC.
- // ReconnectOnFailure must be true.
- // MaxAttempts must be a finite number.
- // SourceId will be complemented with source id prefix and producer type. Can be empty.
- // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent.
- TProducerSettings ProducerSettings;
- unsigned Weight;
-
- explicit TServerWeight(TProducerSettings producerSettings, unsigned weight = 1)
- : ProducerSettings(std::move(producerSettings))
- , Weight(weight)
- {
- Y_VERIFY(Weight > 0);
- }
-
- TServerWeight() = default;
- TServerWeight(const TServerWeight&) = default;
- TServerWeight(TServerWeight&&) = default;
-
- TServerWeight& operator=(const TServerWeight&) = default;
- TServerWeight& operator=(TServerWeight&&) = default;
- };
-
- std::vector<TServerWeight> ServerWeights;
- size_t MinimumWorkingDcsCount = 1; // Minimum Dcs to start and not to die.
- TString SourceIdPrefix; // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent.
-};
-
-struct TConsumerSettings {
- TVector<TString> Topics;
- bool ReadMirroredPartitions = true;
-
- // Discovers all clusters where topic is presented and create consumer that reads from all of them.
- // Conflicts with ReadMirroredPartitions option (should be set to "no").
- // Will ignore the following options:
- // ReconnectOnFailure (forced to "yes")
- // MaxAttempts (forced to infinity == std::numeric_limits<unsigned>::max())
- // Server.UseLogbrokerCDS (forced to "Use")
- //
- // Also see description of option ReconnectOnFailure.
- bool ReadFromAllClusterSources = false;
-
- TServerSetting Server;
- std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider();
- TString ClientId;
-
- bool UseLockSession = false;
- ui32 PartsAtOnce = 0;
-
- bool Unpack = true;
- bool SkipBrokenChunks = false;
-
- ui64 MaxMemoryUsage = DEFAULT_MEMORY_USAGE;
- ui32 MaxInflyRequests = DEFAULT_READS_INFLY;
-
- // read settings
- ui32 MaxCount = 0; // zero means unlimited
- ui32 MaxSize = DEFAULT_BATCH_SIZE; // zero means unlimited
- ui32 MaxTimeLagMs = 0; // zero means unlimited
- ui64 ReadTimestampMs = 0; // read data from this timestamp
-
- ui32 MaxUncommittedCount = 0; // zero means unlimited
- ui64 MaxUncommittedSize = 0; // zero means unlimited
-
- TVector<ui32> PartitionGroups; // groups to read
-
- bool PreferLocalProxy = false;
- bool BalanceRightNow = false;
- bool CommitsDisabled = false;
-
- // Should we reconnect when session breaks?
- //
- // Keep in mind that it is possible to receive
- // message duplicates when reconnection occures.
- bool ReconnectOnFailure = false;
-
- //
- // Settings for reconnection
- //
-
- // In case of lost connection consumer will try
- // to reconnect and resend requests <= MaxAttempts times.
- // Then consumer will be dead.
- unsigned MaxAttempts = std::numeric_limits<unsigned>::max();
-
- // Time delay before next consumer reconnection attempt.
- // The second attempt will be in 2 * ReconnectionDelay,
- // the third - in 3 * ReconnectionDelay, etc. Maximum value
- // of delay is MaxReconnectionDelay.
- TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY;
- TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY;
- TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT;
-
- bool DisableCDS = false; // only for grpcv1 mode
-
- bool UseV2RetryPolicyInCompatMode = false;
-};
-
-struct TProcessorSettings {
- ui64 MaxOriginDataMemoryUsage;
- ui64 MaxProcessedDataMemoryUsage;
- TDuration SourceIdIdleTimeout;
-
- std::shared_ptr<ICredentialsProvider> CredentialsProvider;
-
- //UseLockSession parameter will be ignored
- TConsumerSettings ConsumerSettings;
-
- //Topic and SourceId pararameters will be ignored
- TProducerSettings ProducerSettings;
-
- TProcessorSettings()
- : MaxOriginDataMemoryUsage(DEFAULT_MEMORY_USAGE)
- , MaxProcessedDataMemoryUsage(DEFAULT_MEMORY_USAGE)
- , SourceIdIdleTimeout(TDuration::Hours(1))
- , CredentialsProvider(CreateInsecureCredentialsProvider())
- {
- ConsumerSettings.CredentialsProvider = ProducerSettings.CredentialsProvider = CredentialsProvider;
- }
-};
-
-class TData {
-public:
- // empty: not for passing to producer
- TData() = default;
-
- // implicit constructor from source data to be encoded later by producer with default codec from settings
- // or explicitly specified one
- TData(TString sourceData, ECodec codec = ECodec::DEFAULT, TInstant timestamp = TInstant::Now())
- : TData(timestamp, sourceData, codec == ECodec::RAW ? sourceData : TString(), codec, sourceData.size())
- {
- Y_VERIFY(!SourceData.empty());
- Y_VERIFY(timestamp != TInstant::Zero());
- }
-
- TData(TString sourceData, TInstant timestamp)
- : TData(std::move(sourceData), ECodec::DEFAULT, timestamp)
- {
- }
-
- // construct already encoded TData
- static TData Encoded(TString encodedData, ECodec codec, TInstant timestamp = TInstant::Now(), ui32 originalSize = 0) {
- Y_VERIFY(!encodedData.empty());
- Y_VERIFY(codec != ECodec::RAW && codec != ECodec::DEFAULT);
- Y_VERIFY(timestamp != TInstant::Zero());
- return TData(timestamp, TString(encodedData), encodedData, codec, originalSize == 0 ? encodedData.size() * 10 : originalSize);
- }
-
- // construct user defined raw data that shouldn't be encoded by producer
- static TData Raw(TString rawData, TInstant timestamp = TInstant::Now()) {
- Y_VERIFY(!rawData.empty());
- Y_VERIFY(timestamp != TInstant::Zero());
- return TData(timestamp, rawData, rawData, ECodec::RAW, rawData.size());
- }
-
- TData(const TData&) = default;
- TData(TData&&) = default;
-
- TData& operator=(const TData&) = default;
- TData& operator=(TData&&) = default;
-
- ECodec GetCodecType() const {
- return Codec;
- }
-
- bool IsEncoded() const {
- return !EncodedData.empty();
- }
-
- const TString& GetEncodedData() const {
- Y_VERIFY(IsEncoded());
- return EncodedData;
- }
-
- // gets some data for error report
- const TString& GetSourceData() const {
- return SourceData;
- }
-
- TInstant GetTimestamp() const {
- return Timestamp;
- }
-
- bool Empty() const {
- return GetTimestamp() == TInstant::Zero();
- }
-
- // encoding of data by default codec or data codec that is not encoded
- // if data's codec is default, uses defaultCodec parameter,
- // otherwise uses data's codec
- static TData Encode(TData data, ECodec defaultCodec, int quality = -1);
-
- ui32 GetOriginalSize() const {
- return OriginalSize;
- }
-
- // special function to make raw enconding that doesn't need cpu time.
- static TData MakeRawIfNotEncoded(TData data);
-
- bool operator==(const TData& data) const {
- return Codec == data.Codec
- && Timestamp == data.Timestamp
- && SourceData == data.SourceData
- && EncodedData == data.EncodedData;
- }
-
- bool operator!=(const TData& data) const {
- return !operator==(data);
- }
-
-private:
- TData(TInstant timestamp, TString sourceData, TString encodedData, ECodec codec, ui32 originalSize)
- : Timestamp(timestamp)
- , SourceData(std::move(sourceData))
- , EncodedData(std::move(encodedData))
- , Codec(codec)
- , OriginalSize(originalSize)
- {
- }
-
- static THolder<IOutputStream> CreateCoder(ECodec codec, TData& data, int quality);
-
-private:
- TInstant Timestamp;
- TString SourceData;
- TString EncodedData;
- ECodec Codec = ECodec::DEFAULT;
- ui32 OriginalSize = 0;
-};
-
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt
deleted file mode 100644
index ec1f7fb67f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-v2-ut_utils)
-target_compile_options(cpp-v2-ut_utils PRIVATE
- -DUSE_CURRENT_UDF_ABI_VERSION
-)
-target_link_libraries(cpp-v2-ut_utils PUBLIC
- contrib-libs-cxxsupp
- yutil
- deprecated-cpp-v2
- yndx-grpc_services-persqueue
- yndx-persqueue-msgbus_server
- cpp-grpc-server
- cpp-testing-unittest
- core-client-server
- ydb-core-testlib
- library-persqueue-topic_parser_public
-)
-target_sources(cpp-v2-ut_utils PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp
-)
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp
deleted file mode 100644
index 6f201f39f5..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp
+++ /dev/null
@@ -1,263 +0,0 @@
-#include "data_writer.h"
-
-namespace NPersQueue::NTests {
-using namespace NKikimr;
-
-void TPQDataWriter::Read(
- const TString& topic, const TString& clientId, const TString& ticket, bool error, bool checkACL,
- bool useBatching, bool onlyCreate
-) {
-
- grpc::ClientContext context;
-
- auto stream = StubP_->ReadSession(&context);
- UNIT_ASSERT(stream);
-
- // Send initial request.
- TReadRequest req;
- TReadResponse resp;
-
- req.MutableInit()->AddTopics(topic);
-
- req.MutableInit()->SetClientId(clientId);
- req.MutableInit()->SetProxyCookie(ProxyCookie);
- if (!ticket.empty()) {
- req.MutableCredentials()->SetTvmServiceTicket(ticket);
- }
-
- if (useBatching) {
- req.MutableInit()->SetProtocolVersion(TReadRequest::Batching);
- }
-
- if (!stream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- //TODO[komels]: why this leads to timeout?
- //Server.AnnoyingClient->GetClientInfo({topic}, clientId, true);
- UNIT_ASSERT(stream->Read(&resp));
- if (error) {
- UNIT_ASSERT(resp.HasError());
- return;
- }
- UNIT_ASSERT_C(resp.HasInit(), resp);
-
- if (onlyCreate)
- return;
-
- for (ui32 i = 0; i < 11; ++i) {
- TReadRequest req;
-
- req.MutableRead()->SetMaxCount(1);
-
- if (!stream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- Server.AnnoyingClient->AlterTopic(FullTopicName, i < 10 ? 2 : 3);
-
- }
-
- if (checkACL) {
- NACLib::TDiffACL acl;
- acl.RemoveAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, clientId + "@" BUILTIN_ACL_DOMAIN);
- Server.AnnoyingClient->ModifyACL("/Root/PQ", FullTopicName, acl.SerializeAsString());
-
- TReadRequest req;
- req.MutableRead()->SetMaxCount(1);
- if (!stream->Write(req)) {
- ythrow yexception() << "write fail";
- }
-
- UNIT_ASSERT(stream->Read(&resp));
- UNIT_ASSERT(resp.HasError() && resp.GetError().GetCode() == NPersQueue::NErrorCode::EErrorCode::ACCESS_DENIED);
- return;
- }
- Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true);
- for (ui32 i = 0; i < 11; ++i) {
- TReadResponse resp;
-
- UNIT_ASSERT(stream->Read(&resp));
-
- if (useBatching) {
- UNIT_ASSERT(resp.HasBatchedData());
- UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().PartitionDataSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).BatchSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).MessageDataSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).GetMessageData(0).GetOffset(), i);
- } else {
- UNIT_ASSERT(resp.HasData());
- UNIT_ASSERT_VALUES_EQUAL(resp.GetData().MessageBatchSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).MessageSize(), 1);
- UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).GetMessage(0).GetOffset(), i);
- }
- }
- //TODO: check here that read will never done UNIT_ASSERT(!stream->Read(&resp));
- {
- for (ui32 i = 1; i < 11; ++i) {
- TReadRequest req;
-
- req.MutableCommit()->AddCookie(i);
-
- if (!stream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- }
- ui32 i = 1;
- while (i <= 10) {
- TReadResponse resp;
-
- UNIT_ASSERT(stream->Read(&resp));
- Cerr << resp << "\n";
- UNIT_ASSERT(resp.HasCommit());
- UNIT_ASSERT(resp.GetCommit().CookieSize() > 0);
- for (ui32 j = 0; j < resp.GetCommit().CookieSize(); ++j) {
- UNIT_ASSERT( resp.GetCommit().GetCookie(j) == i);
- ++i;
- UNIT_ASSERT(i <= 11);
- }
- }
- Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true);
- }
-}
-
-void TPQDataWriter::WaitWritePQServiceInitialization() {
- TWriteRequest req;
- TWriteResponse resp;
- while (true) {
- grpc::ClientContext context;
-
- auto stream = StubP_->WriteSession(&context);
- UNIT_ASSERT(stream);
- req.MutableInit()->SetTopic(ShortTopicName);
- req.MutableInit()->SetSourceId("12345678");
- req.MutableInit()->SetProxyCookie(ProxyCookie);
-
- if (!stream->Write(req)) {
- continue;
- }
- UNIT_ASSERT(stream->Read(&resp));
- if (resp.GetError().GetCode() == NPersQueue::NErrorCode::INITIALIZING) {
- Sleep(TDuration::MilliSeconds(50));
- } else {
- break;
- }
- }
-}
-
-ui32 TPQDataWriter::InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step) {
- TWriteRequest req;
- TWriteResponse resp;
-
- grpc::ClientContext context;
-
- auto stream = StubP_->WriteSession(&context);
-
- UNIT_ASSERT(stream);
- req.MutableInit()->SetTopic(ShortTopicName);
- req.MutableInit()->SetSourceId(sourceId);
- req.MutableInit()->SetPartitionGroup(pg);
- req.MutableInit()->SetProxyCookie(ProxyCookie);
-
- UNIT_ASSERT(stream->Write(req));
- UNIT_ASSERT(stream->Read(&resp));
- Cerr << "Init result: " << resp << "\n";
- //TODO: ensure topic creation - proxy already knows about new partitions, but tablet - no!
- if (!success) {
- UNIT_ASSERT(resp.HasError());
- return 0;
- } else {
- if (!resp.HasInit() && step < 5) {
- Sleep(TDuration::MilliSeconds(100));
- return InitSession(sourceId, pg, success, step + 1);
- }
- UNIT_ASSERT(resp.HasInit());
- return resp.GetInit().GetPartition();
- }
- return 0;
-}
-
-ui32 TPQDataWriter::WriteImpl(
- const TString& topic, const TVector<TString>& data, bool error, const TString& ticket, bool batch
-) {
- grpc::ClientContext context;
-
- auto stream = StubP_->WriteSession(&context);
- UNIT_ASSERT(stream);
-
- // Send initial request.
- TWriteRequest req;
- TWriteResponse resp;
-
- req.MutableInit()->SetTopic(topic);
- req.MutableInit()->SetSourceId(SourceId_);
- req.MutableInit()->SetProxyCookie(ProxyCookie);
- if (!ticket.empty())
- req.MutableCredentials()->SetTvmServiceTicket(ticket);
- auto item = req.MutableInit()->MutableExtraFields()->AddItems();
- item->SetKey("key");
- item->SetValue("value");
-
- if (!stream->Write(req)) {
- ythrow yexception() << "write fail";
- }
-
- ui32 part = 0;
-
- UNIT_ASSERT(stream->Read(&resp));
-
- if (!error) {
- UNIT_ASSERT_C(resp.HasInit(), resp);
- UNIT_ASSERT_C(!resp.GetInit().GetSessionId().empty(), resp);
- part = resp.GetInit().GetPartition();
- } else {
- Cerr << resp << "\n";
- UNIT_ASSERT(resp.HasError());
- return 0;
- }
-
- // Send data requests.
- Flush(data, stream, ticket, batch);
-
- Flush(data, stream, ticket, batch);
-
- Flush(data, stream, ticket, batch);
-
- Flush(data, stream, ticket, batch);
-
- //will cause only 4 answers in stream->Read - third call will fail, not blocks
- stream->WritesDone();
-
- UNIT_ASSERT(!stream->Read(&resp));
-
- auto status = stream->Finish();
- UNIT_ASSERT(status.ok());
- return part;
-}
-
-ui64 TPQDataWriter::ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const {
- auto ci = meta.find("cookie");
- if (ci == meta.end()) {
- ythrow yexception() << "server didn't provide the cookie";
- } else {
- return FromString<ui64>(TStringBuf(ci->second.data(), ci->second.size()));
- }
-}
-
-void TPQDataWriter::InitializeChannel() {
- Channel_ = grpc::CreateChannel("[::1]:" + ToString(Server.GrpcPort), grpc::InsecureChannelCredentials());
- Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_);
-
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- NKikimrClient::TResponse response;
- auto status = Stub_->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- ProxyCookie = response.GetProxyCookie();
- Channel_ = grpc::CreateChannel(
- "[" + response.GetProxyName() + "]:" + ToString(Server.GrpcPort),
- grpc::InsecureChannelCredentials()
- );
- StubP_ = NPersQueue::PersQueueService::NewStub(Channel_);
-}
-} // namespace NKikimr::NPersQueueTests
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h
deleted file mode 100644
index c4e46ab08e..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h
+++ /dev/null
@@ -1,117 +0,0 @@
-#pragma once
-
-#include "test_server.h"
-
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <grpc++/client_context.h>
-#include <grpc++/create_channel.h>
-
-namespace NPersQueue::NTests {
-
-class TPQDataWriter {
-public:
- TPQDataWriter(
- const TString& defaultTopicFullName, const TString& defaultShortTopicName,
- const TString& sourceId, TTestServer& server
- )
- : FullTopicName(defaultTopicFullName)
- , ShortTopicName(defaultShortTopicName)
- , SourceId_(sourceId)
- , Server(server)
- {
- InitializeChannel();
- WaitWritePQServiceInitialization();
- }
-
- void Read(const TString& topic, const TString& clientId, const TString& ticket = "", bool error = false,
- bool checkACL = false, bool useBatching = false, bool onlyCreate = false);
-
- void WaitWritePQServiceInitialization();
-
- ui32 InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step = 0);
-
- ui32 Write(const TString& topic, const TString& data, bool error = false, const TString& ticket = "") {
- return WriteImpl(topic, {data}, error, ticket, false);
- }
-
- ui32 WriteBatch(const TString& topic, const TVector<TString>& data, bool error = false,
- const TString& ticket = "") {
- return WriteImpl(topic, data, error, ticket, true);
- }
-
-private:
- ui32 WriteImpl(const TString& topic, const TVector<TString>& data, bool error, const TString& ticket,
- bool batch);
-
- template<typename S>
- void Flush(const TVector<TString>& data, S& stream, const TString& ticket, bool batch) {
- TWriteRequest request;
- TWriteResponse response;
-
- TVector<ui64> allSeqNo;
- if (batch) {
- for (const TString& d: data) {
- ui64 seqNo = AtomicIncrement(SeqNo_);
- allSeqNo.push_back(seqNo);
- auto *mutableData = request.MutableDataBatch()->AddData();
- mutableData->SetSeqNo(seqNo);
- mutableData->SetData(d);
- }
- } else {
- ui64 seqNo = AtomicIncrement(SeqNo_);
- allSeqNo.push_back(seqNo);
- request.MutableData()->SetSeqNo(seqNo);
- request.MutableData()->SetData(JoinSeq("\n", data));
- }
- if (!ticket.empty()) {
- request.MutableCredentials()->SetTvmServiceTicket(ticket);
- }
-
- Cerr << "request: " << request << Endl;
- if (!stream->Write(request)) {
- ythrow yexception() << "write fail";
- }
-
- UNIT_ASSERT(stream->Read(&response));
- if (batch) {
- UNIT_ASSERT_C(response.HasAckBatch(), response);
- UNIT_ASSERT_VALUES_EQUAL(data.size(), response.GetAckBatch().AckSize());
- for (size_t i = 0; i < data.size(); ++i) {
- const auto& ack = response.GetAckBatch().GetAck(i);
- UNIT_ASSERT(!ack.GetAlreadyWritten());
- UNIT_ASSERT(!ack.HasStat());
- UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[i]);
- }
- UNIT_ASSERT(response.GetAckBatch().HasStat());
- } else {
- const auto& ack = response.GetAck();
- UNIT_ASSERT(!ack.GetAlreadyWritten());
- UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[0]);
- UNIT_ASSERT(ack.HasStat());
- }
- }
-
- ui64 ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const;
-
- void InitializeChannel();
-
-private:
- TString FullTopicName;
- TString ShortTopicName;
- const TString SourceId_;
-
- TTestServer& Server;
-
- TAtomic SeqNo_ = 1;
-
- //! Сетевой канал взаимодействия с proxy-сервером.
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_;
- std::unique_ptr<PersQueueService::Stub> StubP_;
-
- ui64 ProxyCookie = 0;
-};
-} // NKikimr::NPersQueueTests
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h
deleted file mode 100644
index 9f0712933a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h
+++ /dev/null
@@ -1,323 +0,0 @@
-#pragma once
-#include "test_server.h"
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-#include <ydb/library/persqueue/topic_parser_public/topic_parser.h>
-#include <library/cpp/logger/log.h>
-#include <util/system/tempfile.h>
-
-#define TEST_CASE_NAME (this->Name_)
-
-namespace NPersQueue {
-class SDKTestSetup {
-protected:
- TString TestCaseName;
-
- THolder<TTempFileHandle> NetDataFile;
- THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters;
- TString LocalDC = "dc1";
- TTestServer Server = TTestServer(false /* don't start */);
-
- TLog Log = TLog("cerr");
-
- TPQLibSettings PQLibSettings;
- THolder<TPQLib> PQLib;
-
-public:
- SDKTestSetup(const TString& testCaseName, bool start = true)
- : TestCaseName(testCaseName)
- {
- InitOptions();
- if (start) {
- Start();
- }
- }
-
- void InitOptions() {
- Log.SetFormatter([testCaseName = TestCaseName](ELogPriority priority, TStringBuf message) {
- return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl;
- });
- Server.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::Max());
- // Default TTestServer value for 'MaxReadCookies' is 10. With this value the tests are flapping with two errors:
- // 1. 'got more than 10 unordered cookies to commit 12'
- // 2. 'got more than 10 uncommitted reads'
- Server.ServerSettings.PQConfig.Clear();
- Server.ServerSettings.PQConfig.SetEnabled(true);
- Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1);
- Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1);
- Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true);
- SetNetDataViaFile("::1/128\t" + GetLocalCluster());
-
- auto seed = TInstant::Now().MicroSeconds();
- // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable
- Log << TLOG_INFO << "Random seed for debugging is " << seed;
- std::srand(seed);
- }
-
- void Start(bool waitInit = true, bool addBrokenDatacenter = !std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) {
- Server.StartServer(false);
- //Server.EnableLogs({NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY});
- Server.AnnoyingClient->InitRoot();
- if (DataCenters.empty()) {
- THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> dataCenters;
- dataCenters.emplace("dc1", NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true});
- if (addBrokenDatacenter) {
- dataCenters.emplace("dc2", NKikimr::NPersQueueTests::TPQTestClusterInfo{"dc2.logbroker.yandex.net", false});
- }
- Server.AnnoyingClient->InitDCs(dataCenters);
- } else {
- Server.AnnoyingClient->InitDCs(DataCenters, LocalDC);
- }
- Server.AnnoyingClient->InitSourceIds();
- CreateTopic(GetTestTopic(), GetLocalCluster());
- if (waitInit) {
- Server.WaitInit(GetTestTopic());
- }
- PQLibSettings.DefaultLogger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG);
- PQLib = MakeHolder<TPQLib>(PQLibSettings);
- }
-
- THolder<TPQLib>& GetPQLib() {
- return PQLib;
- }
-
- const TPQLibSettings& GetPQLibSettings() const {
- return PQLibSettings;
- }
-
- TString GetTestTopic() const {
- return "topic1";
- }
-
- TString GetTestClient() const {
- return "test-reader";
- }
-
- TString GetTestMessageGroupId() const {
- return "test-message-group-id";
- }
-
- TString GetLocalCluster() const {
- return LocalDC;
- }
-
-
- NGrpc::TServerOptions& GetGrpcServerOptions() {
- return Server.GrpcServerOptions;
- }
-
- void SetNetDataViaFile(const TString& netDataTsv) {
- NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
- NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size());
- NetDataFile->FlushData();
- Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
- }
-
- TProducerSettings GetProducerSettings() const {
- TProducerSettings producerSettings;
- producerSettings.Topic = GetTestTopic();
- producerSettings.SourceId = GetTestMessageGroupId();
- producerSettings.Server = TServerSetting{"localhost", Server.GrpcPort};
- producerSettings.Server.Database = "/Root";
- return producerSettings;
- }
-
- TConsumerSettings GetConsumerSettings() const {
- TConsumerSettings consumerSettings;
- consumerSettings.Topics = {GetTestTopic()};
- consumerSettings.ClientId = GetTestClient();
- consumerSettings.Server = TServerSetting{"localhost", Server.GrpcPort};
- consumerSettings.UseV2RetryPolicyInCompatMode = true;
- return consumerSettings;
- }
-
- TLog& GetLog() {
- return Log;
- }
-
-
- template <class TConsumerOrProducer>
- void Start(const THolder<TConsumerOrProducer>& obj) {
- auto startFuture = obj->Start();
- const auto& initResponse = startFuture.GetValueSync();
- UNIT_ASSERT_C(!initResponse.Response.HasError(), "Failed to start: " << initResponse.Response);
- }
-
- THolder<IConsumer> StartConsumer(const TConsumerSettings& settings) {
- THolder<IConsumer> consumer = GetPQLib()->CreateConsumer(settings);
- Start(consumer);
- return consumer;
- }
-
- THolder<IConsumer> StartConsumer() {
- return StartConsumer(GetConsumerSettings());
- }
-
- THolder<IProducer> StartProducer(const TProducerSettings& settings) {
- THolder<IProducer> producer = GetPQLib()->CreateProducer(settings);
- Start(producer);
- return producer;
- }
-
- THolder<IProducer> StartProducer() {
- return StartProducer(GetProducerSettings());
- }
-
- void WriteToTopic(const TVector<TString>& data, IProducer* producer = nullptr) {
- THolder<IProducer> localProducer;
- if (!producer) {
- localProducer = StartProducer();
- producer = localProducer.Get();
- }
- TVector<NThreading::TFuture<TProducerCommitResponse>> resps;
- for (const TString& d : data) {
- Log << TLOG_INFO << "WriteToTopic: " << d;
- resps.push_back(producer->Write(d));
- }
- for (NThreading::TFuture<TProducerCommitResponse>& r : resps) {
- UNIT_ASSERT_C(!r.GetValueSync().Response.HasError(), r.GetValueSync().Response.GetError());
- }
- }
-
- // Read set of sequences from topic
- void ReadFromTopic(const TVector<TVector<TString>>& data, bool commit = true, IConsumer* consumer = nullptr) {
- THolder<IConsumer> localConsumer;
- if (!consumer) {
- localConsumer = StartConsumer();
- consumer = localConsumer.Get();
- }
- TVector<size_t> positions(data.size()); // Initialy zeroes.
-
- int wholeCount = 0;
- for (const TVector<TString>& seq : data) {
- wholeCount += seq.size();
- }
-
- TSet<ui64> cookies;
-
- auto processCommit = [&](const TConsumerMessage& resp) {
- Log << TLOG_INFO << "ReadFromTopic. Committed: " << resp.Response.GetCommit();
- for (ui64 cookie : resp.Response.GetCommit().GetCookie()) {
- UNIT_ASSERT_VALUES_EQUAL(cookies.erase(cookie), 1);
- }
- };
-
- while (wholeCount > 0) {
- auto event = consumer->GetNextMessage();
- const auto& resp = event.GetValueSync();
- UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response);
- if (!resp.Response.HasData()) {
- if (resp.Response.HasCommit()) {
- processCommit(resp);
- }
- continue;
- }
- Log << TLOG_INFO << "ReadFromTopic. Data: " << resp.Response.GetData();
- UNIT_ASSERT(cookies.insert(resp.Response.GetData().GetCookie()).second);
- for (const auto& batch : resp.Response.GetData().GetMessageBatch()) {
- // find proper sequence
- const TString& firstData = batch.GetMessage(0).GetData();
- size_t seqIndex = 0;
- for (; seqIndex < positions.size(); ++seqIndex) {
- if (positions[seqIndex] >= data[seqIndex].size()) { // Already seen.
- continue;
- }
- size_t& seqPos = positions[seqIndex];
- const TString& expectedData = data[seqIndex][seqPos];
- if (expectedData == firstData) {
- UNIT_ASSERT(batch.MessageSize() <= data[seqIndex].size() - positions[seqIndex]);
- ++seqPos;
- --wholeCount;
- // Match.
- for (size_t msgIndex = 1; msgIndex < batch.MessageSize(); ++msgIndex, ++seqPos, --wholeCount) {
- UNIT_ASSERT_STRINGS_EQUAL(batch.GetMessage(msgIndex).GetData(), data[seqIndex][seqPos]);
- }
- break;
- }
- }
- UNIT_ASSERT_LT_C(seqIndex, positions.size(), resp.Response);
- }
-
- if (commit) {
- consumer->Commit({resp.Response.GetData().GetCookie()});
- }
- }
- while (commit && !cookies.empty()) {
- auto event = consumer->GetNextMessage();
- const auto& resp = event.GetValueSync();
- UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response);
- if (!resp.Response.HasCommit()) {
- continue;
- }
- processCommit(resp);
- }
- UNIT_ASSERT_VALUES_EQUAL(wholeCount, 0);
- }
-
- void SetSingleDataCenter(const TString& name = "dc1") {
- UNIT_ASSERT(
- DataCenters.insert(std::make_pair(
- name,
- NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}
- )).second
- );
- LocalDC = name;
- }
-
- void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) {
- if (DataCenters.empty() && setSelfAsDc) {
- SetSingleDataCenter();
- }
- NKikimr::NPersQueueTests::TPQTestClusterInfo info{
- address,
- enabled
- };
- UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second);
- }
-
- void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) {
- AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc);
- }
-
- void EnableDataCenter(const TString& name) {
- auto iter = DataCenters.find(name);
- UNIT_ASSERT(iter != DataCenters.end());
- Server.AnnoyingClient->UpdateDcEnabled(name, true);
- }
- void DisableDataCenter(const TString& name) {
- auto iter = DataCenters.find(name);
- UNIT_ASSERT(iter != DataCenters.end());
- Server.AnnoyingClient->UpdateDcEnabled(name, false);
- }
-
- void ShutdownGRpc() {
- Server.ShutdownGRpc();
- }
-
- void EnableGRpc() {
- Server.EnableGRpc();
- Server.WaitInit(GetTestTopic());
- }
-
- void KickTablets() {
- for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) {
- Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false);
- Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i);
- }
- }
- void AllowTablets() {
- for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) {
- Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true);
- }
- }
-
- void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) {
- Server.AnnoyingClient->CreateTopicNoLegacy(BuildFullTopicName(topic, cluster), partitionsCount);
- }
-
- void KillPqrb(const TString& topic, const TString& cluster) {
- auto describeResult = Server.AnnoyingClient->Ls(TStringBuilder() << "/Root/PQ/" << BuildFullTopicName(topic, cluster));
- UNIT_ASSERT_C(describeResult->Record.GetPathDescription().HasPersQueueGroup(), describeResult->Record);
- Server.AnnoyingClient->KillTablet(*Server.CleverServer, describeResult->Record.GetPathDescription().GetPersQueueGroup().GetBalancerTabletID());
- }
-};
-}
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp
deleted file mode 100644
index f5d0b63cb3..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-#include "test_pqlib.h"
-
-namespace NPersQueue::NTests {
-
- std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer(
- const TProducerSettings& settings, bool deprecated
- ) {
- auto producer = PQLib->CreateProducer(settings, Logger, deprecated);
- TProducerCreateResponse response = producer->Start().GetValueSync();
- return std::tuple<THolder<IProducer>, TProducerCreateResponse>(std::move(producer), response);
- }
-
- std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer(
- const TString& topic,
- const TString& sourceId,
- std::optional<ui32> partitionGroup,
- std::optional<NPersQueueCommon::ECodec> codec,
- std::optional<bool> reconnectOnFailure,
- bool deprecated
- ) {
- TProducerSettings s;
- s.Server = TServerSetting{"localhost", Server.GrpcPort};
- s.Topic = topic;
- s.SourceId = sourceId;
- if (partitionGroup) {
- s.PartitionGroup = partitionGroup.value();
- }
- if (codec) {
- s.Codec = codec.value();
- }
- if (reconnectOnFailure) {
- s.ReconnectOnFailure = reconnectOnFailure.value();
- }
- return CreateProducer(s, deprecated);
- }
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer(
- const TConsumerSettings& settings
- ) {
- auto consumer = PQLib->CreateConsumer(settings, Logger);
- TConsumerCreateResponse response = consumer->Start().GetValueSync();
- return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response);
- }
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateDeprecatedConsumer(
- const TConsumerSettings& settings
- ) {
- auto consumer = PQLib->CreateConsumer(settings, Logger, true);
- TConsumerCreateResponse response = consumer->Start().GetValueSync();
- return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response);
- }
-
- TConsumerSettings TTestPQLib::MakeSettings(
- const TVector<TString>& topics,
- const TString& consumerName
- ) {
- TConsumerSettings s;
- s.Server = TServerSetting{"localhost", Server.GrpcPort};
- s.ClientId = consumerName;
- s.Topics = topics;
- return s;
- }
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer(
- const TVector<TString>& topics,
- const TString& consumerName,
- std::optional<ui32> maxCount,
- std::optional<bool> useLockSession,
- std::optional<bool> readMirroredPartitions,
- std::optional<bool> unpack,
- std::optional<ui32> maxInflyRequests,
- std::optional<ui32> maxMemoryUsage
- ) {
- auto s = MakeSettings(topics, consumerName);
- if (maxCount) {
- s.MaxCount = maxCount.value();
- }
- if (unpack) {
- s.Unpack = unpack.value();
- }
- if (readMirroredPartitions) {
- s.ReadMirroredPartitions = readMirroredPartitions.value();
- }
- if (maxInflyRequests) {
- s.MaxInflyRequests = maxInflyRequests.value();
- }
- if (maxMemoryUsage) {
- s.MaxMemoryUsage = maxMemoryUsage.value();
- }
- if (useLockSession) {
- s.UseLockSession = useLockSession.value();
- }
- return CreateConsumer(s);
- }
-} // namespace NPersQueue::NTests \ No newline at end of file
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h
deleted file mode 100644
index 8e6fb6832d..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#pragma once
-#include "test_server.h"
-
-namespace NPersQueue::NTests {
-
-class TTestPQLib {
-public:
- TTestPQLib(TTestServer& server)
- : Logger(new TCerrLogger(DEBUG_LOG_LEVEL))
- , Server(server)
- {
- TPQLibSettings settings;
- settings.DefaultLogger = Logger;
- PQLib = MakeHolder<TPQLib>(settings);
- }
-
- std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer(
- const TProducerSettings& settings, bool deprecated
- );
-
- std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer(
- const TString& topic, const TString& sourceId, std::optional<ui32> partitionGroup = {},
- std::optional<NPersQueueCommon::ECodec> codec = ECodec::RAW, std::optional<bool> reconnectOnFailure = {},
- bool deprecated = false);
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer(const TConsumerSettings& settings);
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateDeprecatedConsumer(const TConsumerSettings& settings);
-
- TConsumerSettings MakeSettings(const TVector<TString>& topics, const TString& consumerName = "user");
-
- std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer(
- const TVector<TString>& topics, const TString& consumerName = "user", std::optional<ui32> maxCount = {},
- std::optional<bool> useLockSession = {}, std::optional<bool> readMirroredPartitions = {},
- std::optional<bool> unpack = {}, std::optional<ui32> maxInflyRequests = {},
- std::optional<ui32> maxMemoryUsage = {});
-
-private:
- TIntrusivePtr<TCerrLogger> Logger;
- TTestServer& Server;
- THolder<TPQLib> PQLib;
-};
-} // namespace NPersQueue::NTests \ No newline at end of file
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp
deleted file mode 100644
index 6cd1e8094a..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include "test_server.h"
-
-#include <ydb/core/client/server/msgbus_server_pq_metacache.h>
-#include <kikimr/yndx/grpc_services/persqueue/persqueue.h>
-#include <kikimr/yndx/persqueue/msgbus_server/read_session_info.h>
-
-namespace NPersQueue {
-
-const TVector<NKikimrServices::EServiceKikimr> TTestServer::LOGGED_SERVICES = {
- NKikimrServices::PERSQUEUE,
- NKikimrServices::PQ_METACACHE,
- NKikimrServices::PQ_READ_PROXY,
- NKikimrServices::PQ_WRITE_PROXY,
- NKikimrServices::PQ_MIRRORER,
-};
-
-void TTestServer::PatchServerSettings() {
- ServerSettings.RegisterGrpcService<NKikimr::NGRpcService::TGRpcPersQueueService>(
- "pq",
- NKikimr::NMsgBusProxy::CreatePersQueueMetaCacheV2Id()
- );
- ServerSettings.SetPersQueueGetReadSessionsInfoWorkerFactory(
- std::make_shared<NKikimr::NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerWithPQv0Factory>()
- );
-}
-
-void TTestServer::StartIfNeeded(bool start) {
- if (start) {
- StartServer();
- }
-}
-
-
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h
deleted file mode 100644
index 090d9aa47f..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h
+++ /dev/null
@@ -1,129 +0,0 @@
-#pragma once
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-#include <ydb/core/testlib/test_pq_client.h>
-#include <library/cpp/grpc/server/grpc_server.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/system/tempfile.h>
-
-namespace NPersQueue {
-
-static constexpr int DEBUG_LOG_LEVEL = 7;
-
-class TTestServer {
-public:
- TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing())
- : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>()))
- , Port(PortManager->GetPort(2134))
- , GrpcPort(PortManager->GetPort(2135))
- , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort))
- , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort))
- , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) })
- , PQLib(new TPQLib(PQLibSettings))
- {
- PatchServerSettings();
- StartIfNeeded(start);
- }
-
- TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true)
- : PortManager(MakeSimpleShared<TPortManager>())
- , Port(PortManager->GetPort(2134))
- , GrpcPort(PortManager->GetPort(2135))
- , ServerSettings(settings)
- , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort))
- , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) })
- , PQLib(new TPQLib(PQLibSettings))
- {
- ServerSettings.Port = Port;
- ServerSettings.SetGrpcPort(GrpcPort);
- PatchServerSettings();
- StartIfNeeded(start);
- }
-
- void StartServer(bool doClientInit = true) {
- PrepareNetDataFile();
- CleverServer = MakeHolder<NKikimr::Tests::TServer>(ServerSettings);
- CleverServer->EnableGRpc(GrpcServerOptions);
- AnnoyingClient = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(ServerSettings, GrpcPort);
- EnableLogs(LOGGED_SERVICES);
- if (doClientInit) {
- AnnoyingClient->FullInit();
- }
- }
-
- void ShutdownGRpc() {
- CleverServer->ShutdownGRpc();
- }
-
- void EnableGRpc() {
- CleverServer->EnableGRpc(GrpcServerOptions);
- }
-
- void ShutdownServer() {
- CleverServer = nullptr;
- }
-
- void RestartServer() {
- ShutdownServer();
- StartServer();
- }
-
- void EnableLogs(const TVector<NKikimrServices::EServiceKikimr> services,
- NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) {
- Y_VERIFY(CleverServer != nullptr, "Start server before enabling logs");
- for (auto s : services) {
- CleverServer->GetRuntime()->SetLogPriority(s, prio);
- }
- }
-
- void WaitInit(const TString& topic) {
- TProducerSettings s;
- s.Topic = topic;
- s.Server = TServerSetting{"localhost", GrpcPort};
- s.SourceId = "src";
-
- while (PQLib->CreateProducer(s, {}, false)->Start().GetValueSync().Response.HasError()) {
- Sleep(TDuration::MilliSeconds(200));
- }
- }
-
- bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") {
- if (NetDataFile)
- return false;
- NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
- NetDataFile->Write(content.Data(), content.Size());
- NetDataFile->FlushData();
- ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
- return true;
- }
-
- void UpdateDC(const TString& name, bool local, bool enabled) {
- AnnoyingClient->UpdateDC(name, local, enabled);
- }
-
-private:
- void PatchServerSettings();
- void StartIfNeeded(bool start);
-
-public:
- TSimpleSharedPtr<TPortManager> PortManager;
- ui16 Port;
- ui16 GrpcPort;
-
- THolder<NKikimr::Tests::TServer> CleverServer;
- NKikimr::Tests::TServerSettings ServerSettings;
- NGrpc::TServerOptions GrpcServerOptions;
- THolder<TTempFileHandle> NetDataFile;
-
- THolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> AnnoyingClient;
-
- TPQLibSettings PQLibSettings;
- THolder<TPQLib> PQLib;
-
- static const TVector<NKikimrServices::EServiceKikimr> LOGGED_SERVICES;
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h
deleted file mode 100644
index 13b4da2772..0000000000
--- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h
+++ /dev/null
@@ -1,101 +0,0 @@
-#pragma once
-#include <util/generic/ptr.h>
-#include <util/generic/size_literals.h>
-#include <library/cpp/threading/chunk_queue/queue.h>
-#include <util/generic/overloaded.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-namespace NPersQueue {
-
-using namespace NThreading;
-using namespace NYdb::NPersQueue;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-//using namespace NPersQueue::V1;
-
-template <class TPQLibObject>
-void DestroyAndWait(THolder<TPQLibObject>& object) {
- if (object) {
- auto isDead = object->IsDead();
- object = nullptr;
- isDead.GetValueSync();
- }
-}
-
-inline bool GrpcV1EnabledByDefault() {
- static const bool enabled = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED");
- return enabled;
-}
-
-class TCallbackCredentialsProvider : public ICredentialsProvider {
- std::function<void(NPersQueue::TCredentials*)> Callback;
-public:
- TCallbackCredentialsProvider(std::function<void(NPersQueue::TCredentials*)> callback)
- : Callback(std::move(callback))
- {}
-
- void FillAuthInfo(NPersQueue::TCredentials* authInfo) const {
- Callback(authInfo);
- }
-};
-
-struct TWriteResult {
- bool Ok = false;
- // No acknowledgement is expected from a writer under test
- bool NoWait = false;
- TString ResponseDebugString = TString();
-};
-
-struct TAcknowledgableMessage {
- TString Value;
- ui64 SequenceNumber;
- TInstant CreatedAt;
- TPromise<TWriteResult> AckPromise;
-};
-
-class IClientEventLoop {
-protected:
- std::atomic_bool MayStop;
- std::atomic_bool MustStop;
- bool Stopped = false;
- std::unique_ptr<TThread> Thread;
- TLog Log;
-
-public:
- IClientEventLoop()
- : MayStop()
- , MustStop()
- , MessageBuffer()
- {}
-
- void AllowStop() {
- MayStop = true;
- }
-
- void WaitForStop() {
- if (!Stopped) {
- Log << TLOG_INFO << "Wait for writer to die on itself";
- Thread->Join();
- Log << TLOG_INFO << "Client write event loop stopped";
- }
- Stopped = true;
- }
-
- virtual ~IClientEventLoop() {
- MustStop = true;
- if (!Stopped) {
- Log << TLOG_INFO << "Wait for client write event loop to stop";
- Thread->Join();
- Log << TLOG_INFO << "Client write event loop stopped";
- }
- Stopped = true;
- }
-
- TManyOneQueue<TAcknowledgableMessage> MessageBuffer;
-
-};
-
-} // namespace NPersQueue
diff --git a/kikimr/public/README.md b/kikimr/public/README.md
deleted file mode 100644
index b7422d385d..0000000000
--- a/kikimr/public/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Public APIs and Client SDKs for Yandex Database (YDB)
-
-`api` -- public API for YDB is implemented via gRPC framework;
-`lib` -- some common libraries used by public SDKs and YDB core itself;
-`sdk` -- programming language specific client SDK. If you are building an application on top of YDB these SDKs are for you;
-`tools` -- developer tools: recipe to be used in tests, program to setup local YDB cluster.
diff --git a/kikimr/public/sdk/cpp/README.md b/kikimr/public/sdk/cpp/README.md
deleted file mode 100644
index adc7f4e4b5..0000000000
--- a/kikimr/public/sdk/cpp/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-C++ SDK for Yandex Database (YDB)
----
-Officially supported C++ client for YDB.
diff --git a/kikimr/public/sdk/cpp/client/CHANGELOG.md b/kikimr/public/sdk/cpp/client/CHANGELOG.md
deleted file mode 100644
index bd8515352a..0000000000
--- a/kikimr/public/sdk/cpp/client/CHANGELOG.md
+++ /dev/null
@@ -1,60 +0,0 @@
-## 2.2.0 ##
-
-* Disable client query cache by default
-
-## 2.1.0 ##
-
-* Allow c++ sdk to create sessions in different ydb servers.
-
-## 2.0.0 ##
-
-* Remove request migrator feature from c++ sdk.
-
-## 1.6.2 ##
-
-* Fix SSL settings override for client
-
-## 1.6.1 ##
-
-* Do not wrap status in to CLIENT_DISCOVERY_FAILED in case of discovery error
-
-## 1.6.0 ##
-
-* Experimental interface to execute query from credential provider
-
-## 1.5.1 ##
-
-* Do not miss exceptions thrown by lambdas given to RetryOperation() method.
-
-## 1.5.0 ##
-
-* Added support of session graceful shutdown protocol
-
-## 1.4.0 ##
-
-* Support for socket timeout settings
-
-## 1.3.0 ##
-
-* Added user account key support for iam authentication
-
-## 1.2.0 ##
-
-* Added creation from environment support for DriverConfig
-
-## 1.1.0 ##
-
-* Support for grpc channel buffer limit settings
-
-## 1.0.2 ##
-
-* Added connection string support for CommonClientSettings, DriverConfig
-
-## 1.0.1 ##
-
-* Try to bind endpoint to session for
- BeginTransaction, CommitTransaction, RollbackTransaction, KeepAlive, CloseSession, ExplainDataQuery requests
-
-## 1.0.0 ##
-
-* Start initial changelog.
diff --git a/kikimr/public/sdk/cpp/client/iam/CMakeLists.txt b/kikimr/public/sdk/cpp/client/iam/CMakeLists.txt
deleted file mode 100644
index 27a57988b5..0000000000
--- a/kikimr/public/sdk/cpp/client/iam/CMakeLists.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-client-iam)
-target_link_libraries(cpp-client-iam PUBLIC
- contrib-libs-cxxsupp
- yutil
- cloud-iam-v1
- priv-iam-v1
- cpp-grpc-client
- cpp-http-simple
- library-cpp-json
- cpp-threading-atomic
- public-lib-jwt
- client-ydb_types-credentials
-)
-target_sources(cpp-client-iam PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/public/sdk/cpp/client/iam/iam.cpp
-)
diff --git a/kikimr/public/sdk/cpp/client/iam/iam.cpp b/kikimr/public/sdk/cpp/client/iam/iam.cpp
deleted file mode 100644
index ff8f79d4fd..0000000000
--- a/kikimr/public/sdk/cpp/client/iam/iam.cpp
+++ /dev/null
@@ -1,361 +0,0 @@
-#include "iam.h"
-
-#include <cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.pb.h>
-#include <cloud/bitbucket/public-api/yandex/cloud/iam/v1/iam_token_service.grpc.pb.h>
-
-#include <cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.pb.h>
-#include <cloud/bitbucket/private-api/yandex/cloud/priv/iam/v1/iam_token_service.grpc.pb.h>
-
-
-#include <library/cpp/grpc/client/grpc_client_low.h>
-#include <library/cpp/threading/atomic/bool.h>
-#include <library/cpp/threading/future/core/future.h>
-#include <library/cpp/json/json_reader.h>
-#include <library/cpp/http/simple/http_client.h>
-
-#include <util/system/spinlock.h>
-#include <util/stream/file.h>
-#include <util/string/builder.h>
-
-#include <chrono>
-
-using namespace yandex::cloud::iam::v1;
-
-
-using namespace NGrpc;
-
-namespace NYdb {
-
-constexpr TDuration BACKOFF_START = TDuration::MilliSeconds(50);
-constexpr TDuration BACKOFF_MAX = TDuration::Seconds(10);
-
-class TIAMCredentialsProvider : public ICredentialsProvider {
-public:
- TIAMCredentialsProvider(const TIamHost& params)
- : HttpClient_(TSimpleHttpClient(params.Host, params.Port))
- , Request_("/computeMetadata/v1/instance/service-accounts/default/token")
- , NextTicketUpdate_(TInstant::Zero())
- , RefreshPeriod_(params.RefreshPeriod)
- {
- GetTicket();
- }
-
- TStringType GetAuthInfo() const override {
- if (TInstant::Now() >= NextTicketUpdate_) {
- GetTicket();
- }
- return Ticket_;
- }
-
- bool IsValid() const override {
- return true;
- }
-
-private:
- TSimpleHttpClient HttpClient_;
- TStringType Request_;
- mutable TStringType Ticket_;
- mutable TInstant NextTicketUpdate_;
- TDuration RefreshPeriod_;
-
- void GetTicket() const {
- try {
- TStringStream out;
- TSimpleHttpClient::THeaders headers;
- headers["Metadata-Flavor"] = "Google";
- HttpClient_.DoGet(Request_, &out, headers);
- NJson::TJsonValue resp;
- NJson::ReadJsonTree(&out, &resp, true);
-
- auto respMap = resp.GetMap();
-
- if (auto it = respMap.find("access_token"); it == respMap.end())
- ythrow yexception() << "Result doesn't contain access_token";
- else if (TString ticket = it->second.GetStringSafe(); ticket.empty())
- ythrow yexception() << "Got empty ticket";
- else
- Ticket_ = std::move(ticket);
-
- if (auto it = respMap.find("expires_in"); it == respMap.end())
- ythrow yexception() << "Result doesn't contain expires_in";
- else {
- const TDuration expiresIn = TDuration::Seconds(it->second.GetUInteger());
-
- NextTicketUpdate_ = TInstant::Now() + std::max(expiresIn, RefreshPeriod_);
- }
- } catch (...) {
- }
- }
-};
-
-
-template<typename TRequest, typename TResponse, typename TService>
-class TGrpcIamCredentialsProvider : public ICredentialsProvider {
-protected:
- using TRequestFiller = std::function<void(TRequest&)>;
-
-private:
- class TImpl : public std::enable_shared_from_this<TGrpcIamCredentialsProvider<TRequest, TResponse, TService>::TImpl> {
- public:
- TImpl(const TIamEndpoint& iamEndpoint, const TRequestFiller& requestFiller)
- : Client(MakeHolder<NGrpc::TGRpcClientLow>())
- , Connection_(nullptr)
- , Ticket_("")
- , NextTicketUpdate_(TInstant::Zero())
- , IamEndpoint_(iamEndpoint)
- , RequestFiller_(requestFiller)
- , RequestInflight_(false)
- , LastRequestError_("")
- , NeedStop_(false)
- , BackoffTimeout_(BACKOFF_START)
- , Lock_()
- {
- TGRpcClientConfig grpcConf;
- grpcConf.Locator = IamEndpoint_.Endpoint;
- grpcConf.EnableSsl = true;
- Connection_ = THolder<TServiceConnection<TService>>(Client->CreateGRpcServiceConnection<TService>(grpcConf).release());
- }
-
- void UpdateTicket(bool sync = false) {
- with_lock(Lock_) {
- if (NeedStop_ || RequestInflight_) {
- return;
- }
- RequestInflight_ = true;
- }
-
- auto resultPromise = NThreading::NewPromise();
-
- std::shared_ptr<TImpl> self = TGrpcIamCredentialsProvider<TRequest, TResponse, TService>::TImpl::shared_from_this();
-
- auto cb = [self, resultPromise, sync](
- NGrpc::TGrpcStatus&& status, TResponse&& result) mutable {
- self->ProcessIamResponse(std::move(status), std::move(result), sync);
- resultPromise.SetValue();
- };
-
- TRequest req;
-
- RequestFiller_(req);
-
- Connection_->template DoRequest<TRequest, TResponse>(
- std::move(req),
- std::move(cb),
- &TService::Stub::AsyncCreate,
- { {}, {}, IamEndpoint_.RequestTimeout }
- );
-
- if (sync) {
- resultPromise.GetFuture().Wait(2 * IamEndpoint_.RequestTimeout);
- }
- }
-
- TStringType GetTicket() {
- TInstant nextTicketUpdate;
- TString ticket;
- with_lock(Lock_) {
- ticket = Ticket_;
- nextTicketUpdate = NextTicketUpdate_;
- if (ticket.empty())
- ythrow yexception() << "IAM-token not ready yet. " << LastRequestError_;
- }
- if (TInstant::Now() >= nextTicketUpdate) {
- UpdateTicket();
- }
- return ticket;
- }
-
- void Stop() {
- with_lock(Lock_) {
- if (NeedStop_) {
- return;
- }
- NeedStop_ = true;
- }
-
- Client.Reset(); // Will trigger destroy
- }
-
- private:
- void ProcessIamResponse(NGrpc::TGrpcStatus&& status, TResponse&& result, bool sync) {
- if (!status.Ok()) {
- TDuration sleepDuration;
- with_lock(Lock_) {
- LastRequestError_ = TStringBuilder()
- << "Last request error was at " << TInstant::Now()
- << ". GrpcStatusCode: " << status.GRpcStatusCode
- << " Message: \"" << status.Msg
- << "\" internal: " << status.InternalError
- << " iam-endpoint: \"" << IamEndpoint_.Endpoint << "\"";
-
- RequestInflight_ = false;
- sleepDuration = std::min(BackoffTimeout_, BACKOFF_MAX);
- BackoffTimeout_ *= 2;
- }
-
- Sleep(sleepDuration);
-
- UpdateTicket(sync);
- } else {
- with_lock(Lock_) {
- LastRequestError_ = "";
- Ticket_ = result.iam_token();
- RequestInflight_ = false;
- BackoffTimeout_ = BACKOFF_START;
-
- const auto now = Now();
- NextTicketUpdate_ = std::min(
- now + IamEndpoint_.RefreshPeriod,
- TInstant::Seconds(result.expires_at().seconds())
- ) - IamEndpoint_.RequestTimeout;
- NextTicketUpdate_ = std::max(NextTicketUpdate_, now + TDuration::MilliSeconds(100));
- }
- }
- }
-
- private:
-
- THolder<TGRpcClientLow> Client;
- THolder<TServiceConnection<TService>> Connection_;
- TStringType Ticket_;
- TInstant NextTicketUpdate_;
- const TIamEndpoint IamEndpoint_;
- const TRequestFiller RequestFiller_;
- bool RequestInflight_;
- TStringType LastRequestError_;
- bool NeedStop_;
- TDuration BackoffTimeout_;
- TAdaptiveLock Lock_;
- };
-
-public:
- TGrpcIamCredentialsProvider(const TIamEndpoint& endpoint, const TRequestFiller& requestFiller)
- : Impl_(std::make_shared<TImpl>(endpoint, requestFiller))
- {
- Impl_->UpdateTicket(true);
- }
-
- ~TGrpcIamCredentialsProvider() {
- Impl_->Stop();
- }
-
- TStringType GetAuthInfo() const override {
- return Impl_->GetTicket();
- }
-
- bool IsValid() const override {
- return true;
- }
-
-private:
- std::shared_ptr<TImpl> Impl_;
-};
-
-struct TIamJwtParams : TIamEndpoint {
- TJwtParams JwtParams;
-};
-
-template<typename TRequest, typename TResponse, typename TService>
-class TIamJwtCredentialsProvider : public TGrpcIamCredentialsProvider<TRequest, TResponse, TService> {
-public:
- TIamJwtCredentialsProvider(const TIamJwtParams& params)
- : TGrpcIamCredentialsProvider<TRequest, TResponse, TService>(params,
- [jwtParams = params.JwtParams](TRequest& req) {
- req.set_jwt(MakeSignedJwt(jwtParams));
- }) {}
-};
-
-class TIamOAuthCredentialsProvider : public TGrpcIamCredentialsProvider<CreateIamTokenRequest, CreateIamTokenResponse, IamTokenService> {
-public:
- TIamOAuthCredentialsProvider(const TIamOAuth& params)
- : TGrpcIamCredentialsProvider(params,
- [token = params.OAuthToken](CreateIamTokenRequest& req) {
- req.set_yandex_passport_oauth_token(token);
- }) {}
-};
-
-TJwtParams ReadJwtKeyFile(const TString& filename) {
- return ParseJwtParams(TFileInput(filename).ReadAll());
-}
-
-class TIamCredentialsProviderFactory : public ICredentialsProviderFactory {
-public:
- TIamCredentialsProviderFactory(const TIamHost& params): Params_(params) {}
-
- TCredentialsProviderPtr CreateProvider() const final {
- return std::make_shared<TIAMCredentialsProvider>(Params_);
- }
-
- TStringType GetClientIdentity() const final {
- return "IAM_PROVIDER" + ToString((ui64)this);
- }
-
-private:
- TIamHost Params_;
-};
-
-template<typename TRequest, typename TResponse, typename TService>
-class TIamJwtCredentialsProviderFactory : public ICredentialsProviderFactory {
-public:
- TIamJwtCredentialsProviderFactory(const TIamJwtParams& params): Params_(params) {}
-
- TCredentialsProviderPtr CreateProvider() const final {
- return std::make_shared<TIamJwtCredentialsProvider<TRequest, TResponse, TService>>(Params_);
- }
-
- TStringType GetClientIdentity() const final {
- return "IAM_JWT_PROVIDER" + ToString((ui64)this);
- }
-
-private:
- TIamJwtParams Params_;
-};
-
-class TIamOAuthCredentialsProviderFactory : public ICredentialsProviderFactory {
-public:
- TIamOAuthCredentialsProviderFactory(const TIamOAuth& params): Params_(params) {}
-
- TCredentialsProviderPtr CreateProvider() const final {
- return std::make_shared<TIamOAuthCredentialsProvider>(Params_);
- }
-
- TStringType GetClientIdentity() const final {
- return "IAM_OAUTH_PROVIDER" + ToString((ui64)this);
- }
-
-private:
- TIamOAuth Params_;
-};
-
-TCredentialsProviderFactoryPtr CreateIamCredentialsProviderFactory(const TIamHost& params) {
- return std::make_shared<TIamCredentialsProviderFactory>(params);
-}
-
-TCredentialsProviderFactoryPtr CreateIamJwtCredentialsProviderFactoryImpl(TIamJwtParams&& jwtParams, bool usePrivateApi) {
- if (usePrivateApi) {
- return std::make_shared<TIamJwtCredentialsProviderFactory<
- yandex::cloud::priv::iam::v1::CreateIamTokenRequest,
- yandex::cloud::priv::iam::v1::CreateIamTokenResponse,
- yandex::cloud::priv::iam::v1::IamTokenService
- >>(std::move(jwtParams));
- }
- return std::make_shared<TIamJwtCredentialsProviderFactory<CreateIamTokenRequest,
- CreateIamTokenResponse,
- IamTokenService>>(std::move(jwtParams));
-
-}
-
-TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactory(const TIamJwtFilename& params, bool usePrivateApi) {
- TIamJwtParams jwtParams = { params, ReadJwtKeyFile(params.JwtFilename) };
- return CreateIamJwtCredentialsProviderFactoryImpl(std::move(jwtParams), usePrivateApi);
-}
-
-TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactory(const TIamJwtContent& params, bool usePrivateApi) {
- TIamJwtParams jwtParams = { params, ParseJwtParams(params.JwtContent) };
- return CreateIamJwtCredentialsProviderFactoryImpl(std::move(jwtParams), usePrivateApi);
-}
-
-TCredentialsProviderFactoryPtr CreateIamOAuthCredentialsProviderFactory(const TIamOAuth& params) {
- return std::make_shared<TIamOAuthCredentialsProviderFactory>(params);
-}
-} // namespace NYdb
diff --git a/kikimr/public/sdk/cpp/client/iam/iam.h b/kikimr/public/sdk/cpp/client/iam/iam.h
deleted file mode 100644
index 68d6d16db5..0000000000
--- a/kikimr/public/sdk/cpp/client/iam/iam.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-
-#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
-
-#include <ydb/public/lib/jwt/jwt.h>
-#include <util/datetime/base.h>
-
-namespace NYdb {
-
-namespace NIam {
-constexpr TStringBuf DEFAULT_ENDPOINT = "iam.api.cloud.yandex.net";
-
-constexpr TStringBuf DEFAULT_HOST = "169.254.169.254";
-constexpr ui32 DEFAULT_PORT = 80;
-
-constexpr TDuration DEFAULT_REFRESH_PERIOD = TDuration::Hours(1);
-constexpr TDuration DEFAULT_REQUEST_TIMEOUT = TDuration::Seconds(10);
-}
-
-struct TIamHost {
- TString Host = TString(NIam::DEFAULT_HOST);
- ui32 Port = NIam::DEFAULT_PORT;
- TDuration RefreshPeriod = NIam::DEFAULT_REFRESH_PERIOD;
-};
-
-struct TIamEndpoint {
- TString Endpoint = TString(NIam::DEFAULT_ENDPOINT);
- TDuration RefreshPeriod = NIam::DEFAULT_REFRESH_PERIOD;
- TDuration RequestTimeout = NIam::DEFAULT_REQUEST_TIMEOUT;
-};
-
-struct TIamJwtFilename : TIamEndpoint { TString JwtFilename; };
-
-struct TIamJwtContent : TIamEndpoint { TString JwtContent; };
-
-struct TIamOAuth : TIamEndpoint { TString OAuthToken; };
-
-/// Acquire an IAM token using a local metadata service on a virtual machine.
-TCredentialsProviderFactoryPtr CreateIamCredentialsProviderFactory(const TIamHost& params = {});
-
-/// Acquire an IAM token using a JSON Web Token (JWT) file name.
-TCredentialsProviderFactoryPtr CreateIamJwtFileCredentialsProviderFactory(const TIamJwtFilename& params, bool usePrivateApi = false);
-
-/// Acquire an IAM token using JSON Web Token (JWT) contents.
-TCredentialsProviderFactoryPtr CreateIamJwtParamsCredentialsProviderFactory(const TIamJwtContent& param, bool usePrivateApi = false);
-
-// Acquire an IAM token using a user OAuth token.
-TCredentialsProviderFactoryPtr CreateIamOAuthCredentialsProviderFactory(const TIamOAuth& params);
-
-} // namespace NYdb
diff --git a/kikimr/public/sdk/cpp/client/ydb_persqueue/CMakeLists.txt b/kikimr/public/sdk/cpp/client/ydb_persqueue/CMakeLists.txt
deleted file mode 100644
index 9450a17202..0000000000
--- a/kikimr/public/sdk/cpp/client/ydb_persqueue/CMakeLists.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-client-ydb_persqueue INTERFACE)
-target_link_libraries(cpp-client-ydb_persqueue INTERFACE
- contrib-libs-cxxsupp
- yutil
- client-ydb_persqueue-codecs
- cpp-client-ydb_persqueue_core
-)
diff --git a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/CMakeLists.txt b/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/CMakeLists.txt
deleted file mode 100644
index e8ac82ab8c..0000000000
--- a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(client-ydb_persqueue-codecs)
-target_link_libraries(client-ydb_persqueue-codecs PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-streams-lzop
- cpp-streams-zstd
- public-issue-protos
- api-grpc-draft
- api-protos
-)
-target_sources(client-ydb_persqueue-codecs PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.cpp
-)
diff --git a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.cpp b/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.cpp
deleted file mode 100644
index 6c0bdd1215..0000000000
--- a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-#include <library/cpp/streams/lzop/lzop.h>
-#include <library/cpp/streams/zstd/zstd.h>
-#include <util/stream/buffer.h>
-#include <util/stream/zlib.h>
-#include <util/stream/mem.h>
-
-#include "codecs.h"
-
-namespace NYdb::NPersQueue {
-namespace NCompressionDetails {
-
-using TInputStreamVariant = std::variant<std::monostate, TZLibDecompress, TLzopDecompress, TZstdDecompress>;
-
-IInputStream* CreateDecompressorStream(TInputStreamVariant& inputStreamStorage, Ydb::PersQueue::V1::Codec codec, IInputStream* origin) {
- switch (codec) {
- case Ydb::PersQueue::V1::CODEC_GZIP:
- return &inputStreamStorage.emplace<TZLibDecompress>(origin);
- case Ydb::PersQueue::V1::CODEC_LZOP:
- return &inputStreamStorage.emplace<TLzopDecompress>(origin);
- case Ydb::PersQueue::V1::CODEC_ZSTD:
- return &inputStreamStorage.emplace<TZstdDecompress>(origin);
- default:
- //case Ydb::PersQueue::V1::CODEC_RAW:
- //case Ydb::PersQueue::V1::CODEC_UNSPECIFIED:
- throw yexception() << "unsupported codec value : " << ui64(codec);
- }
-}
-
-TString Decompress(const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::MessageData& data) {
- TMemoryInput input(data.data().data(), data.data().size());
- TString result;
- TStringOutput resultOutput(result);
- TInputStreamVariant inputStreamStorage;
- TransferData(CreateDecompressorStream(inputStreamStorage, data.codec(), &input), &resultOutput);
- return result;
-}
-
-
-class TZLibToStringCompressor: private TEmbedPolicy<TBufferOutput>, public TZLibCompress {
-public:
- TZLibToStringCompressor(TBuffer& dst, ZLib::StreamType type, size_t quality)
- : TEmbedPolicy<TBufferOutput>(dst)
- , TZLibCompress(TEmbedPolicy::Ptr(), type, quality)
- {
- }
-};
-
-class TLzopToStringCompressor: private TEmbedPolicy<TBufferOutput>, public TLzopCompress {
-public:
- TLzopToStringCompressor(TBuffer& dst)
- : TEmbedPolicy<TBufferOutput>(dst)
- , TLzopCompress(TEmbedPolicy::Ptr())
- {
- }
-};
-
-class TZstdToStringCompressor: private TEmbedPolicy<TBufferOutput>, public TZstdCompress {
-public:
- TZstdToStringCompressor(TBuffer& dst, int quality)
- : TEmbedPolicy<TBufferOutput>(dst)
- , TZstdCompress(TEmbedPolicy::Ptr(), quality)
- {
- }
-};
-
-THolder<IOutputStream> CreateCoder(ECodec codec, TBuffer& result, int quality) {
- switch (codec) {
- case ECodec::GZIP:
- return MakeHolder<TZLibToStringCompressor>(result, ZLib::GZip, quality >= 0 ? quality : 6);
- case ECodec::LZOP:
- return MakeHolder<TLzopToStringCompressor>(result);
- case ECodec::ZSTD:
- return MakeHolder<TZstdToStringCompressor>(result, quality);
- default:
- Y_FAIL("NOT IMPLEMENTED CODEC TYPE");
- }
-}
-
-
-} // namespace NDecompressionDetails
-
-} // namespace NYdb::NPersQueue
diff --git a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.h b/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.h
deleted file mode 100644
index 5fb5bb466f..0000000000
--- a/kikimr/public/sdk/cpp/client/ydb_persqueue/codecs/codecs.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#pragma once
-#include <util/stream/output.h>
-#include <ydb/public/api/protos/ydb_persqueue_v1.pb.h>
-#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h>
-
-
-namespace NYdb::NPersQueue {
-namespace NCompressionDetails {
-
-
-extern TString Decompress(const Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::DataBatch::MessageData& data);
-
-THolder<IOutputStream> CreateCoder(ECodec codec, TBuffer& result, int quality);
-
-} // namespace NDecompressionDetails
-
-} // namespace NYdb::NPersQueue
-
diff --git a/kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h b/kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h
deleted file mode 100644
index 44ba01d94f..0000000000
--- a/kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h
+++ /dev/null
@@ -1,2 +0,0 @@
-#pragma once
-#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h>
diff --git a/kikimr/yndx/api/grpc/CMakeLists.txt b/kikimr/yndx/api/grpc/CMakeLists.txt
deleted file mode 100644
index adcf9dcf68..0000000000
--- a/kikimr/yndx/api/grpc/CMakeLists.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(api-grpc-yndx)
-set_property(TARGET api-grpc-yndx PROPERTY
- PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
-)
-target_link_libraries(api-grpc-yndx PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
- api-protos-yndx
- api-protos
- contrib-libs-protobuf
-)
-target_proto_messages(api-grpc-yndx PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/grpc/persqueue.proto
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/grpc/ydb_yndx_keyvalue_v1.proto
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/grpc/ydb_yndx_rate_limiter_v1.proto
-)
-target_proto_addincls(api-grpc-yndx
- ./
- ${CMAKE_SOURCE_DIR}/
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(api-grpc-yndx
- --cpp_out=${CMAKE_BINARY_DIR}/
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
-)
-target_proto_plugin(api-grpc-yndx
- grpc_cpp
- grpc_cpp
-)
diff --git a/kikimr/yndx/api/grpc/persqueue.proto b/kikimr/yndx/api/grpc/persqueue.proto
deleted file mode 100644
index 313b42ca3d..0000000000
--- a/kikimr/yndx/api/grpc/persqueue.proto
+++ /dev/null
@@ -1,68 +0,0 @@
-syntax = "proto3";
-
-package NPersQueue;
-
-option java_package = "com.yandex.persqueue";
-option java_outer_classname = "PersqueueGrpc";
-
-import "kikimr/yndx/api/protos/persqueue.proto";
-
-service PersQueueService {
-
- /**
- * Creates Write Session
- * Pipeline:
- * client server
- * Init(Topic, SourceId, ...)
- * ---------------->
- * Init(Partition, MaxSeqNo, ...)
- * <----------------
- * write(data1, seqNo1)
- * ---------------->
- * write(data2, seqNo2)
- * ---------------->
- * ack(seqNo1, offset1, ...)
- * <----------------
- * write(data3, seqNo3)
- * ---------------->
- * ack(seqNo2, offset2, ...)
- * <----------------
- * error(description, errorCode)
- * <----------------
- */
-
- rpc WriteSession(stream WriteRequest) returns (stream WriteResponse);
-
- /**
- * Creates Read Session
- * Pipeline:
- * client server
- * Init(Topics, ClientId, ...)
- * ---------------->
- * Init(SessionId)
- * <----------------
- * read1
- * ---------------->
- * read2
- * ---------------->
- * lock(Topic1,Partition1, ...) - locks and releases are optional
- * <----------------
- * lock(Topic2, Partition2, ...)
- * <----------------
- * release(Topic1, Partition1, ...)
- * <----------------
- * locked(Topic2, Partition2, ...) - client must respond to lock request with this message. Only after this client will start recieving messages from this partition
- * ---------------->
- * read result(data, ...)
- * <----------------
- * commit(cookie1)
- * ---------------->
- * commit result(cookie1)
- * <----------------
- * error(description, errorCode)
- * <----------------
- */
-
- rpc ReadSession(stream ReadRequest) returns (stream ReadResponse);
-
-} \ No newline at end of file
diff --git a/kikimr/yndx/api/grpc/ydb_yndx_keyvalue_v1.proto b/kikimr/yndx/api/grpc/ydb_yndx_keyvalue_v1.proto
deleted file mode 100644
index 7b685f5966..0000000000
--- a/kikimr/yndx/api/grpc/ydb_yndx_keyvalue_v1.proto
+++ /dev/null
@@ -1,43 +0,0 @@
-syntax = "proto3";
-
-package Ydb.Yndx.KeyValue.V1;
-
-option java_package = "com.yandex.ydb.yndx.keyvalue.v1";
-option java_outer_classname = "KeyValueGrpc";
-option java_multiple_files = true;
-
-import "kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto";
-
-// KeyValue tablets provide a simple key-value storage in a low-overhead and easy-to-shoot-your-leg manner.
-// To use KeyValue tablets in an efficient way one must be familiar with the design of both the KeyValue tablet
-// and the Distributed Storage underneath it.
-
-service KeyValueService {
-
- // Create a keyvalue volume by the path and a count of partitions
- rpc CreateVolume(KeyValue.CreateVolumeRequest) returns (KeyValue.CreateVolumeResponse);
-
- // Drop the keyvalue volume by the path
- rpc DropVolume(KeyValue.DropVolumeRequest) returns (KeyValue.DropVolumeResponse);
-
- // List partitions of keyvalue volume in the local node.
- rpc ListLocalPartitions(KeyValue.ListLocalPartitionsRequest) returns (KeyValue.ListLocalPartitionsResponse);
-
- // Obtains an exclusive lock for the tablet.
- rpc AcquireLock(KeyValue.AcquireLockRequest) returns (KeyValue.AcquireLockResponse);
-
- // Performs one or more actions that modify the state of the tablet as an atomic transaction.
- rpc ExecuteTransaction(KeyValue.ExecuteTransactionRequest) returns (KeyValue.ExecuteTransactionResponse);
-
- // Reads value stored in the item with the key specified.
- rpc Read(KeyValue.ReadRequest) returns (KeyValue.ReadResponse);
-
- // Reads a list of items with the keys in the range specified.
- rpc ReadRange(KeyValue.ReadRangeRequest) returns (KeyValue.ReadRangeResponse);
-
- // List existed items with the keys in the range specified.
- rpc ListRange(KeyValue.ListRangeRequest) returns (KeyValue.ListRangeResponse);
-
- // Gets storage channel status of the tablet.
- rpc GetStorageChannelStatus(KeyValue.GetStorageChannelStatusRequest) returns (KeyValue.GetStorageChannelStatusResponse);
-}
diff --git a/kikimr/yndx/api/grpc/ydb_yndx_rate_limiter_v1.proto b/kikimr/yndx/api/grpc/ydb_yndx_rate_limiter_v1.proto
deleted file mode 100644
index 9e0c2e97dd..0000000000
--- a/kikimr/yndx/api/grpc/ydb_yndx_rate_limiter_v1.proto
+++ /dev/null
@@ -1,35 +0,0 @@
-syntax = "proto3";
-
-package Ydb.Yndx.RateLimiter.V1;
-
-option java_package = "com.yandex.ydb.yndx.rate_limiter.v1";
-option java_outer_classname = "YndxRateLimiterGrpc";
-option java_multiple_files = true;
-
-import "kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto";
-
-// Service that implements distributed rate limiting and accounting.
-//
-// To use rate limiter functionality you need an existing coordination node.
-
-service YndxRateLimiterService {
- // Control plane API
-
- // Create a new resource in existing coordination node.
- rpc CreateResource(CreateResourceRequest) returns (CreateResourceResponse);
-
- // Update a resource in coordination node.
- rpc AlterResource(AlterResourceRequest) returns (AlterResourceResponse);
-
- // Delete a resource from coordination node.
- rpc DropResource(DropResourceRequest) returns (DropResourceResponse);
-
- // List resources in given coordination node.
- rpc ListResources(ListResourcesRequest) returns (ListResourcesResponse);
-
- // Describe properties of resource in coordination node.
- rpc DescribeResource(DescribeResourceRequest) returns (DescribeResourceResponse);
-
- // Take units for usage of a resource in coordination node.
- rpc AcquireResource(AcquireResourceRequest) returns (AcquireResourceResponse);
-}
diff --git a/kikimr/yndx/api/protos/CMakeLists.txt b/kikimr/yndx/api/protos/CMakeLists.txt
deleted file mode 100644
index 3f858c162c..0000000000
--- a/kikimr/yndx/api/protos/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(api-protos-yndx)
-target_link_libraries(api-protos-yndx PUBLIC
- contrib-libs-cxxsupp
- yutil
- api-protos
- tools-enum_parser-enum_serialization_runtime
- contrib-libs-protobuf
-)
-target_proto_messages(api-protos-yndx PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/protos/persqueue.proto
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto
-)
-generate_enum_serilization(api-protos-yndx
- ${CMAKE_BINARY_DIR}/kikimr/yndx/api/protos/persqueue.pb.h
- INCLUDE_HEADERS
- kikimr/yndx/api/protos/persqueue.pb.h
-)
-target_proto_addincls(api-protos-yndx
- ./
- ${CMAKE_SOURCE_DIR}/
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(api-protos-yndx
- --cpp_out=${CMAKE_BINARY_DIR}/
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
-)
diff --git a/kikimr/yndx/api/protos/persqueue.proto b/kikimr/yndx/api/protos/persqueue.proto
deleted file mode 100644
index e8da4c4a36..0000000000
--- a/kikimr/yndx/api/protos/persqueue.proto
+++ /dev/null
@@ -1,335 +0,0 @@
-syntax = "proto3";
-import "google/protobuf/descriptor.proto";
-import "ydb/public/api/protos/draft/persqueue_common.proto";
-
-package NPersQueue;
-
-option java_package = "com.yandex.ydb.persqueue";
-option cc_enable_arenas = true;
-
-extend google.protobuf.FileOptions {
- bool GenerateYaStyle = 66677;
-}
-
-message Path {
- // Path of object (topic/consumer).
- string path = 1;
-}
-
-// WRITE REQUEST
-
-message KeyValue {
- string key = 1;
- string value = 2;
-}
-
-message MapType {
- repeated KeyValue items = 1;
-}
-
-/**
- * Request for write session. Contains one of :
- * Init - consists of initialization info - Topic, SourceId and so on
- * Data - data to be writen
- * DataBatch - batch of data to be written
- */
-message WriteRequest {
- message Init {
- string topic = 1;
- bytes source_id = 2;
-
- MapType extra_fields = 7; //server and file inside here
-
- uint64 proxy_cookie = 8; //cookie provided by ChooseProxy request //change to bytes
-
- uint32 partition_group = 12; //Group to write to - 0 means any;
-
- string version = 999; //must be filled by client lib
- }
-
- message Data {
- uint64 seq_no = 1;
- bytes data = 2;
- uint64 create_time_ms = 3; //timestamp in ms
- NPersQueueCommon.ECodec codec = 4;
- uint32 uncompressed_size = 5;
- }
-
- message DataBatch {
- repeated Data data = 1;
- }
-
- oneof request {
- //init must be sent as first message
- Init init = 1;
- Data data = 2;
- DataBatch data_batch = 3;
- }
-
- NPersQueueCommon.Credentials credentials = 20;
-}
-
-/**
- * Response for write session. Contains one of :
- * Error - in any error state - grpc errors, session dies, incorrect Init request and so on
- * Init - contains SessionId of created session, MaxSeqNo and Partition
- * Ack - acknowlegment of storing corresponding message
- * AckBatch - acknowlegment of storing corresponding message batch
- */
-message WriteResponse {
- message Init {
- uint64 max_seq_no = 1;
- string session_id = 2;
- uint32 partition = 3;
- string topic = 4;
- }
-
- message Stat {
- uint32 write_time_ms = 1;
- uint32 total_time_in_partition_queue_ms = 2;
- uint32 partition_quoted_time_ms = 3;
- uint32 topic_quoted_time_ms = 4;
- }
-
- message Ack {
- uint64 seq_no = 1;
- uint64 offset = 2;
- bool already_written = 3;
-
- Stat stat = 4; //not filled in batch case
- }
-
- message AckBatch {
- Stat stat = 2; //common statistics for batch storing
-
- repeated Ack ack = 1;
- }
-
- oneof response {
- Init init = 1;
- Ack ack = 2;
- AckBatch ack_batch = 4;
- NPersQueueCommon.Error error = 3;
- }
-}
-
-// READ REQUEST
-
-/**
- * Request for read session. Contains one of :
- * Init - contains of Topics to be readed, ClientId and other metadata
- * Read - request for read batch. Contains of restrictments for result - MaxSize, MaxCount and so on
- * Commit - request for commit some read batches. Contains corresponding cookies
- * Locked - comfirming to server that client is ready to get data from partition from concreet offset
- */
-message ReadRequest {
- enum EProtocolVersion {
- Base = 0; // Base protocol version
- Batching = 1; // Client supports more effective batching structs (TBatchedData instead of TData)
- ReadParamsInInit = 2; // Client sets read params in Init request
- }
-
- message Init {
- repeated string topics = 1;
- bool read_only_local = 2; // ReadOnlyLocal=false - read mirrored topics from other clusters too; will be renamed to read_only_original
-
- string client_id = 4;
- bool clientside_locks_allowed = 5; //if true then partitions Lock signal will be sent from server,
- //and reads from partitions will began only after Locked signal recieved by server from client
-
- uint64 proxy_cookie = 6; //cookie provided by ChooseProxy request
-
- bool balance_partition_right_now = 8; //if set then do not wait for commits from client on data from partition in case of balancing
-
- repeated uint32 partition_groups = 9; //Groups to be read - if empty then read from all of them
-
- uint32 idle_timeout_sec = 10; //TODO: do we need it?
-
- uint32 commit_interval_ms = 12; // How often server must commit data. If client sends commits faster,
- // then server will hold them in order to archive corresponding rate; zero means server default = 1sec
-
- // Read request params
- uint32 max_read_messages_count = 14; // Max messages to give to client in one read request
- uint32 max_read_size = 15; // Max size in bytes to give to client in one read request
- uint32 max_read_partitions_count = 16; // 0 means not matters // Maximum partitions count to give to client in one read request
- uint32 max_time_lag_ms = 17; // Read data only with time lag less than or equal to specified
- uint64 read_timestamp_ms = 18; // Read data only after this timestamp
-
- bool commits_disabled = 19; // Client will never commit
-
- string version = 999; //must be filled by client lib
-
- // Protocol version to let server know about new features that client supports
- uint32 protocol_version = 13; // version must be integer (not enum) because client may be newer than server
- }
-
- message Read {
- // It is not allowed to change these parameters.
- // They will be removed in future from TRead structure.
- uint32 max_count = 1;
- uint32 max_size = 2;
- uint32 partitions_at_once = 3; //0 means not matters
- uint32 max_time_lag_ms = 5;
- uint64 read_timestamp_ms = 6; //read data only after this timestamp
- }
-
- message StartRead {
- string topic = 1;
- uint32 partition = 2;
-
- uint64 read_offset = 3; //skip upto this position; if committed position is bigger, then do nothing
- bool verify_read_offset = 4; //if true then check that committed position is <= ReadOffset; otherwise it means error in client logic
- uint64 generation = 5;
- uint64 commit_offset = 6; //all messages BEFORE this position are processed by client
- }
-
- message Commit {
- repeated uint64 cookie = 1;
- }
-
- message Status {
- uint64 generation = 1;
- string topic = 2;
- uint32 partition = 3;
- }
-
- oneof request {
- //init must be sent as first message
- Init init = 1;
- Read read = 2;
- StartRead start_read = 3;
- Commit commit = 4;
- Status status = 5;
- }
-
- NPersQueueCommon.Credentials credentials = 20;
-}
-
-
-message MessageMeta {
- bytes source_id = 1;
- uint64 seq_no = 2;
- uint64 create_time_ms = 3;
- uint64 write_time_ms = 4;
-
- MapType extra_fields = 7;
- NPersQueueCommon.ECodec codec = 8;
- string ip = 9;
- uint32 uncompressed_size = 10;
-}
-
-/**
- * Response for read session. Contains one of :
- * Error - in any error state - grpc errors, session dies, incorrect Init request and so on
- * Init - contains SessionId of created session
- * Data - result of read, contains of messages batch and cookie
- * Commit - acknowlegment for commit
- * Lock - informs client that server is ready to read data from corresponding partition
- * Release - informs client that server will not get data from this partition in future read results, unless other Lock-Locked conversation will be done
- */
-
-message ReadResponse {
- message Init {
- string session_id = 2; //for debug only
- }
-
- message Data {
- message Message {
- MessageMeta meta = 1; //SeqNo ...
- bytes data = 2;
- //unique value for clientside deduplication - Topic:Partition:Offset
- uint64 offset = 3;
- bytes broken_packed_data = 4; // TODO: move to pqlib
- }
-
- message MessageBatch {
- string topic = 1;
- uint32 partition = 2;
- repeated Message message = 3;
- }
-
- repeated MessageBatch message_batch = 1;
- uint64 cookie = 2; //Cookie to be committed by server
- }
-
- message BatchedData {
- message MessageData {
- NPersQueueCommon.ECodec codec = 2;
-
- uint64 offset = 3; //unique value for clientside deduplication - Topic:Partition:Offset
- uint64 seq_no = 4;
-
- uint64 create_time_ms = 5;
- uint64 uncompressed_size = 6;
-
- bytes data = 1;
- }
-
- message Batch {
- bytes source_id = 2;
- MapType extra_fields = 3;
- uint64 write_time_ms = 4;
- string ip = 5;
-
- repeated MessageData message_data = 1;
- }
-
- message PartitionData {
- string topic = 2;
- uint32 partition = 3;
-
- repeated Batch batch = 1;
- }
-
- uint64 cookie = 2; //Cookie to be committed by server
-
- repeated PartitionData partition_data = 1; //not greater than one PartitionData for each partition
- }
-
- message Lock {
- string topic = 1;
- uint32 partition = 2;
-
- uint64 read_offset = 3; //offset to read from
- uint64 end_offset = 4; //know till this time end offset
- uint64 generation = 5;
- }
-
- message Release {
- string topic = 1;
- uint32 partition = 2;
- bool can_commit = 3; //if CanCommit=false then you can not store progress of processing data for that partition at server;
- //all commits will have no effect for this partition
- //if you rely on committing offsets then just drop all data for this partition without processing - another session will get them later
- //if CanCommit=true and you are relying on committing offsets - you can process all data for this partition you got,
- //commit cookies and be sure that no other session will ever get this data
- uint64 generation = 4;
- }
-
- message Commit {
- repeated uint64 cookie = 1; //for debug purposes only
- }
-
- // Response for status requst.
- message PartitionStatus {
- uint64 generation = 1;
- string topic = 2;
- uint32 partition = 3;
-
- uint64 committed_offset = 4;
- uint64 end_offset = 5;
- uint64 write_watermark_ms = 6;
- }
-
- oneof response {
- Init init = 1;
- Data data = 2;
- BatchedData batched_data = 7;
- NPersQueueCommon.Error error = 3;
- Lock lock = 4;
- Release release = 5;
- Commit commit = 6;
- PartitionStatus partition_status = 8;
- }
-}
-
diff --git a/kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto b/kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto
deleted file mode 100644
index 6782a1bfe5..0000000000
--- a/kikimr/yndx/api/protos/ydb_yndx_keyvalue.proto
+++ /dev/null
@@ -1,460 +0,0 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package Ydb.Yndx.KeyValue;
-
-option java_package = "com.yandex.ydb.yndx.rate_limiter";
-option java_outer_classname = "YndxkeyValueProtos";
-option java_multiple_files = true;
-
-import "ydb/public/api/protos/ydb_operation.proto";
-
-//
-// KeyValue API.
-//
-
-message Flags {
- // Free disk space is low.
- bool disk_space_cyan = 1;
-
- // Free disk space is low, it is recommended to stop writing additional data.
- bool disk_space_light_yellow_move = 2;
- bool disk_space_yellow_stop = 3;
-
- // Free disk space is very low, clients must stop writing additional data.
- bool disk_space_light_orange = 4;
- bool disk_space_orange = 5;
-
- // Free disk space is extremely low, operations other than deletion may not be performed.
- bool disk_space_red = 6;
-
- // No free disk space available.
- bool disk_space_black = 7;
-}
-
-
-message Statuses {
- enum Status {
- STATUS_UNSPECIFIED = 0;
- STATUS_SUCCESS = 1;
- STATUS_NO_DATA = 2;
- STATUS_ERROR = 3;
- STATUS_OVERRUN = 4;
- }
-}
-
-
-message StorageChannel {
- // XXX
- Statuses.Status status = 1;
-
- // Storage channel index.
- uint32 storage_channel = 2;
-
- // If present, contains the status flags of the storage channel. Empty if status flags could not be obtained.
- optional Flags status_flags = 3;
-}
-
-
-message Priorities {
- enum Priority {
- PRIORITY_UNSPECIFIED = 0;
-
- // High priority for user-initiated operations.
- PRIORITY_REALTIME = 1;
-
- // Low prioroty for background system activity.
- PRIORITY_BACKGROUND = 2;
- }
-}
-
-
-message KVRange {
- // The first bound of the range of the keys
- // If no one is assigned then specify in order for the range to begin from the lowest key
- oneof from_bound {
- // Specify in order for the range to include the key specified
- string from_key_inclusive = 1;
- // Specify in order for the range not to include the key specified
- string from_key_exclusive = 2;
- }
-
- // The second bound of the range of the keys
- // If no one is assigned then specify in order for the range to end to the highest keys
- oneof to_bound {
- // Specify in order for the range to include the key specified
- string to_key_inclusive = 3;
- // Specify in order for the range not to include the key specified
- string to_key_exclusive = 4;
- }
-}
-
-
-message AcquireLockRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-}
-
-
-message AcquireLockResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message AcquireLockResult {
- // The generation of the lock to provide as an argument to all the operaions the user performs with the tablet.
- uint64 lock_generation = 1;
-}
-
-message ExecuteTransactionRequest {
- message Command {
- message Rename {
- // The key to change.
- string old_key = 1;
-
- // The new key to change the old key to.
- string new_key = 2;
- }
- message Concat {
- // Keys to use as the source for the concatenation.
- repeated string input_keys = 1;
-
- // New key to use for the result of the concatenation
- string output_key = 2;
-
- // Input keys are deleted after the concatenation by default. In order to keep both the inputs and the
- // output, set keep_inputs to true.
- bool keep_inputs = 3;
- }
-
- // Makes a copy of a range of key-value pairs. New keys are formed by removing a prefix and/or prepending a new
- // prefix. For example, copy of the key-value pairs [{aaabc,1},{aaaef,2}] can be stripped of the 'aa' prefix and
- // prepended with the 'x' so that the new pairs are [{xabc, 1}, {xaef, 2}].
- message CopyRange {
- // The range of keys to copy
- KVRange range = 1;
-
- // For each source key that begins with the prefix_to_remove, that prefix is removed from the new key before
- // prepending it with the prefix_to_add. Acts as filter if not empty.
- string prefix_to_remove = 2;
-
- // The prefix_to_add prefix is prepended to each new key.
- string prefix_to_add = 3;
- }
- message Write {
- enum Tactic {
- TACTIC_UNSPECIFIED = 0;
-
- // Write minimum required redundant data. Does not affect storage durability.
- TACTIC_MAX_THROUGHPUT = 1;
-
- // Write additional redundant data to more disks to reduce operation duration. Does not affect storage
- // durability, but will use additional space.
- TACTIC_MIN_LATENCY = 2;
- }
- // Key of the key-value pair to write.
- string key = 1;
-
- // Value of the key-value pair to write.
- bytes value = 2;
-
- // Storage channel to write the value to. Channel numbers begin with 1 and may go up to approximately 250
- // (depends on the channel configuration of each tablet).
- // Channel 1 is called the INLINE channel (value is stored in the index table).
- // Channel 2 is called the MAIN channel (value is stored as a separate blob in the Distributed Storage).
- // Channels 1 and 2 are available for all tablets.
- // If the storage channel specified is not configured for the tablet, the value is stored in
- // channel 2 (the MAIN channel).
- uint32 storage_channel = 3; // (default = 0 is same as 2 or MAIN)
-
- // Priority to use for the Distributed Storage Get operation. Has no effect for the 1st (inline) storage
- // channel. Defaults to PRIORITY_UNSPECIFIED which interpreted like PRIORITY_REALTIME.
- Priorities.Priority priority = 4;
-
- // Tactic to use for the Distributed Storage Put operation. Has no effect for the 1st (inline) storage
- // channel. Defaults to TACTIC_UNSPECIFIED which interpreted like TACTIC_MAX_THROUGHPUT.
- Tactic tactic = 5;
- }
- message DeleteRange {
- // The range of keys to delete
- KVRange range = 1;
- }
-
- oneof action {
- // Deletes key-value pairs with keys in the range specified.
- DeleteRange delete_range = 1;
-
- // Changes the key of a key-value pair.
- Rename rename = 2;
-
- // Creates a copy of key-value pairs with keys in the range specified by removin and/or prepending a prefix
- // specified to each key.
- CopyRange copy_range = 3;
-
- // Creates a new key-value pair with key specified by concatenating values of multiple other key-value pairs
- // with keys specified.
- Concat concat = 4;
-
- // Creates a new key-value pair with key and value specified.
- Write write = 5;
- }
- }
-
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-
- // Generation of the exclusive lock obtained for the tablet as a result of an AcquireLock call.
- uint64 lock_generation = 4;
-
- // Commands to execute as a single atomic transaction. The order of execution of commands is the same as the order
- // of commands in the ExecuteTransactionRequest. Order of execution of different transactions is not specified.
- repeated Command commands = 5;
-}
-
-message ExecuteTransactionResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ExecuteTransactionResult {
- repeated StorageChannel storage_channel = 1;
-}
-
-message ReadRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-
- // Generation of the exclusive lock obtained for the tablet as a result of an AcquireLock call.
- uint64 lock_generation = 4;
-
- // Key of the key-value pair to read.
- string key = 5;
-
- // Offset in bytes from the beginning of the value to read data from.
- uint64 offset = 6;
-
- // Size of the data to read in bytes. 0 means "read to the end of the value".
- uint64 size = 7;
-
- // Result protobuf size limit. If not 0, overrides the default one only with a smaller value.
- uint64 limit_bytes = 8;
-
- // Priority to use for the Distributed Storage Get operation. Has no effect for the 1st (inline) storage
- // channel. Defaults to PRIORITY_UNSPECIFIED which interpreted like PRIORITY_REALTIME.
- Priorities.Priority priority = 9;
-}
-
-message ReadResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ReadResult {
- // The key of the requested key-value pair
- string requested_key = 1;
-
- // Offset in bytes from the beginning of the value requested
- uint64 requested_offset = 2;
-
- // Size of the data requested
- uint64 requested_size = 3;
-
- // The bytes of the requested part of the value of the requested key-value pair
- bytes value = 4;
-
- // XXX
- string msg = 5;
-
- Statuses.Status status = 6;
-}
-
-message ReadRangeRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-
- // Generation of the exclusive lock obtained for the tablet as a result of an AcquireLock call.
- uint64 lock_generation = 4;
-
- // The range of keys to read
- KVRange range = 5;
-
- // Result protobuf size limit. If not 0, overrides the default one only with a smaller value.
- uint64 limit_bytes = 6;
-
- // Priority to use for the Distributed Storage Get operation. Has no effect for the 1st (inline) storage
- // channel. Defaults to PRIORITY_UNSPECIFIED which interpreted like PRIORITY_REALTIME.
- Priorities.Priority priority = 7;
-}
-
-message ReadRangeResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ReadRangeResult {
- message KeyValuePair {
- // The key of the key-value pair.
- string key = 1;
-
- // The value of the key-value pair. Present only if the request was performed with include_data set to true.
- bytes value = 2;
-
- // Full size of the value of the key-value pair.
- uint32 value_size = 3;
-
- // Unix time of the creation of the key-value pair (in ms).
- uint64 creation_unix_time = 4;
-
- // Contains the index of the actualy used storage channel. The actually used storage channel may differ from
- // the value specified in the write request for example if there were no such storage channel at the moment
- // of execution of the write command.
- // For values created as a result of a concatenation or a copy of such values, the storage channel of the first
- // part of the value is specified.
- uint32 storage_channel = 5; // Returns the _actual_ storage channel
-
- Statuses.Status status = 6;
- }
- Statuses.Status status = 1;
-
- // List of the key-value pairs and metadata requested.
- repeated KeyValuePair pair = 2;
-}
-
-message ListRangeRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-
- // Generation of the exclusive lock obtained for the tablet as a result of an AcquireLock call.
- uint64 lock_generation = 4;
-
- // The range of keys to read
- KVRange range = 5;
-
- // Result protobuf size limit. If not 0, overrides the default one only with a smaller value.
- uint64 limit_bytes = 6;
-
- // Priority to use for the Distributed Storage Get operation. Has no effect for the 1st (inline) storage
- // channel. Defaults to PRIORITY_UNSPECIFIED which interpreted like PRIORITY_REALTIME.
- Priorities.Priority priority = 7;
-}
-
-message ListRangeResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ListRangeResult {
- message KeyInfo {
- // The key of the key-value pair.
- string key = 1;
-
- // Full size of the value of the key-value pair.
- uint32 value_size = 2;
-
- // Unix time of the creation of the key-value pair (in ms).
- uint64 creation_unix_time = 3;
-
- // Contains the index of the actualy used storage channel. The actually used storage channel may differ from
- // the value specified in the write request for example if there were no such storage channel at the moment
- // of execution of the write command.
- // For values created as a result of a concatenation or a copy of such values, the storage channel of the first
- // part of the value is specified.
- uint32 storage_channel = 4; // Returns the _actual_ storage channel
- }
- Statuses.Status status = 1;
-
- // List of the key-value pairs and metadata requested.
- repeated KeyInfo key = 2;
-}
-
-message GetStorageChannelStatusRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint64 partition_id = 3;
-
- // Generation of the exclusive lock obtained for the tablet as a result of an AcquireLock call.
- uint64 lock_generation = 4;
-
- // Storage channel index.
- repeated uint32 storage_channel = 5;
-}
-
-message GetStorageChannelStatusResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message GetStorageChannelStatusResult {
- repeated StorageChannel storage_channel = 1;
-}
-
-message CreateVolumeRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
- uint32 channel_profile_id = 3;
- uint32 partition_count = 4;
-}
-
-message CreateVolumeResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message CreateVolumeResult {
-}
-
-message DropVolumeRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
-}
-
-message DropVolumeResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message DropVolumeResult {
-}
-
-message ListLocalPartitionsRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path to the volume
- string path = 2;
-
- // If it's zero than it used local node.
- uint64 node_id = 3;
-}
-
-message ListLocalPartitionsResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ListLocalPartitionsResult {
- string requested_path = 1;
- uint64 node_id = 2;
-
- repeated uint64 partition_ids = 3;
-}
diff --git a/kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto b/kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto
deleted file mode 100644
index 10ba9627da..0000000000
--- a/kikimr/yndx/api/protos/ydb_yndx_rate_limiter.proto
+++ /dev/null
@@ -1,273 +0,0 @@
-syntax = "proto3";
-option cc_enable_arenas = true;
-
-package Ydb.Yndx.RateLimiter;
-
-option java_package = "com.yandex.ydb.yndx.rate_limiter";
-option java_outer_classname = "YndxRateLimiterProtos";
-option java_multiple_files = true;
-
-import "ydb/public/api/protos/ydb_operation.proto";
-
-//
-// Rate Limiter control API.
-//
-
-//
-// Resource properties.
-//
-
-message AccountingConfig {
- // Account consumed resources and send billing metrics.
- // Default value is false (not inherited).
- bool enabled = 1;
-
- // Period to report consumption history from clients to kesus
- // Default value is inherited from parent or equals 5000 ms for root.
- uint64 report_period_ms = 2;
-
- // Consumption history period that is sent in one message to accounting actor.
- // Default value is inherited from parent or equals 1000 ms for root.
- uint64 account_period_ms = 3;
-
- // Time window to collect data from every client.
- // Any client account message that is `collect_period` late is discarded (not accounted or billed).
- // Default value is inherited from parent or equals 30 seconds for root.
- uint64 collect_period_sec = 4;
-
- // Provisioned consumption limit in units per second.
- // Effective value is limited by corresponding `max_units_per_second`.
- // Default value is 0 (not inherited).
- double provisioned_units_per_second = 5;
-
- // Provisioned allowed burst equals `provisioned_coefficient * provisioned_units_per_second` units.
- // Effective value is limited by corresponding PrefetchCoefficient.
- // Default value is inherited from parent or equals 60 for root.
- double provisioned_coefficient = 6;
-
- // On-demand allowed burst equals `overshoot_coefficient * prefetch_coefficient * max_units_per_second` units.
- // Should be greater or equal to 1.0
- // Default value is inherited from parent or equals 1.1 for root
- double overshoot_coefficient = 7;
-
- // Billing metric description.
- message Metric {
- // Send this metric to billing.
- // Default value is false (not inherited).
- bool enabled = 1;
-
- // Billing metric period (aligned to hour boundary).
- // Default value is inherited from parent or equals 60 seconds for root.
- uint64 billing_period_sec = 2;
-
- // Billing metric JSON fields (inherited from parent if not set)
- string version = 3;
- string schema = 5;
- string cloud_id = 6;
- string folder_id = 7;
- string resource_id = 8;
- string source_id = 9;
- }
-
- // Consumption within provisioned limit.
- // Informative metric that should be sent to billing (not billed).
- Metric provisioned = 8;
-
- // Consumption that exceeds provisioned limit is billed as on-demand.
- Metric on_demand = 9;
-
- // Consumption that exceeds even on-demand limit.
- // Normally it is free and should not be billed.
- Metric overshoot = 10;
-}
-
-// Settings for hierarchical deficit round robin (HDRR) algorithm.
-message HierarchicalDrrSettings {
- // Resource consumption speed limit.
- // Value is required for root resource.
- // 0 is equivalent to not set.
- // Must be nonnegative.
- double max_units_per_second = 1;
-
- // Maximum burst size of resource consumption across the whole cluster
- // divided by max_units_per_second.
- // Default value is 1.
- // This means that maximum burst size might be equal to max_units_per_second.
- // 0 is equivalent to not set.
- // Must be nonnegative.
- double max_burst_size_coefficient = 2;
-
- // Prefetch in local bucket up to prefetch_coefficient*max_units_per_second units (full size).
- // Default value is inherited from parent or 0.2 for root.
- // Disables prefetching if any negative value is set
- // (It is useful to avoid bursts in case of large number of local buckets).
- double prefetch_coefficient = 3;
-
- // Prefetching starts if there is less than prefetch_watermark fraction of full local bucket left.
- // Default value is inherited from parent or 0.75 for root.
- // Must be nonnegative and less than or equal to 1.
- double prefetch_watermark = 4;
-}
-
-// Rate limiter resource description.
-message Resource {
- // Resource path. Elements are separated by slash.
- // The first symbol is not slash.
- // The first element is root resource name.
- // Resource path is the path of resource inside coordination node.
- string resource_path = 1;
-
- oneof type {
- // Settings for Hierarchical DRR algorithm.
- HierarchicalDrrSettings hierarchical_drr = 2;
- }
-
- AccountingConfig accounting_config = 3;
-}
-
-//
-// CreateResource method.
-//
-
-message CreateResourceRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // Resource properties.
- Resource resource = 3;
-}
-
-message CreateResourceResponse {
- // Holds CreateResourceResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message CreateResourceResult {
-}
-
-//
-// AlterResource method.
-//
-
-message AlterResourceRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // New resource properties.
- Resource resource = 3;
-}
-
-message AlterResourceResponse {
- // Holds AlterResourceResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message AlterResourceResult {
-}
-
-//
-// DropResource method.
-//
-
-message DropResourceRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // Path of resource inside a coordination node.
- string resource_path = 3;
-}
-
-message DropResourceResponse {
- // Holds DropResourceResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message DropResourceResult {
-}
-
-//
-// ListResources method.
-//
-
-message ListResourcesRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // Path of resource inside a coordination node.
- // May be empty.
- // In that case all root resources will be listed.
- string resource_path = 3;
-
- // List resources recursively.
- bool recursive = 4;
-}
-
-message ListResourcesResponse {
- // Holds ListResourcesResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message ListResourcesResult {
- repeated string resource_paths = 1;
-}
-
-//
-// DescribeResource method.
-//
-
-message DescribeResourceRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // Path of resource inside a coordination node.
- string resource_path = 3;
-}
-
-message DescribeResourceResponse {
- // Holds DescribeResourceResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message DescribeResourceResult {
- Resource resource = 1;
-}
-
-//
-// AcquireResource method.
-//
-
-message AcquireResourceRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Path of a coordination node.
- string coordination_node_path = 2;
-
- // Path of resource inside a coordination node.
- string resource_path = 3;
-
- oneof units {
- // Request resource's units for usage.
- uint64 required = 4;
-
- // Actually used resource's units by client.
- uint64 used = 5;
- }
-}
-
-message AcquireResourceResponse {
- // Holds AcquireResourceResult in case of successful call.
- Ydb.Operations.Operation operation = 1;
-}
-
-message AcquireResourceResult {
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/CMakeLists.txt b/kikimr/yndx/grpc_services/persqueue/CMakeLists.txt
deleted file mode 100644
index 5b00f89620..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/CMakeLists.txt
+++ /dev/null
@@ -1,38 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yndx-grpc_services-persqueue)
-target_link_libraries(yndx-grpc_services-persqueue PUBLIC
- contrib-libs-cxxsupp
- yutil
- api-grpc-yndx
- api-protos-yndx
- yndx-persqueue-read_batch_converter
- ydb-core-base
- core-client-server
- ydb-core-grpc_services
- core-mind-address_classification
- ydb-core-persqueue
- core-persqueue-events
- core-persqueue-writer
- ydb-core-protos
- ydb-library-aclib
- library-persqueue-topic_parser
- services-lib-actors
- services-lib-sharding
- ydb-services-persqueue_v1
-)
-target_sources(yndx-grpc_services-persqueue PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/grpc_pq_read_actor.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/grpc_pq_write_actor.cpp
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/grpc_services/persqueue/persqueue.cpp
-)
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_actor.h b/kikimr/yndx/grpc_services/persqueue/grpc_pq_actor.h
deleted file mode 100644
index 155c803d0d..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_actor.h
+++ /dev/null
@@ -1,928 +0,0 @@
-#pragma once
-
-#include "grpc_pq_session.h"
-#include "ydb/core/client/server/msgbus_server_pq_metacache.h"
-#include "ydb/core/client/server/msgbus_server_persqueue.h"
-
-#include <ydb/core/base/events.h>
-#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <ydb/core/protos/grpc_pq_old.pb.h>
-#include <ydb/core/protos/pqconfig.pb.h>
-#include <ydb/core/persqueue/writer/source_id_encoding.h>
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-
-#include <library/cpp/actors/core/hfunc.h>
-
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <ydb/core/persqueue/events/global.h>
-#include <ydb/core/persqueue/writer/writer.h>
-#include <ydb/core/persqueue/percentile_counter.h>
-#include <ydb/core/base/appdata.h>
-#include <ydb/core/base/tablet_pipe.h>
-#include <ydb/core/tx/tx_proxy/proxy.h>
-#include <ydb/public/lib/base/msgbus_status.h>
-#include <ydb/core/kqp/kqp.h>
-
-#include <ydb/core/base/ticket_parser.h>
-#include <ydb/services/lib/actors/type_definitions.h>
-#include <ydb/services/persqueue_v1/actors/read_init_auth_actor.h>
-#include <ydb/services/persqueue_v1/actors/read_session_actor.h>
-
-
-#include <util/generic/guid.h>
-#include <util/system/compiler.h>
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-
-static inline TVector<TEvTicketParser::TEvAuthorizeTicket::TEntry> GetTicketParserEntries(const TString& dbId, const TString& folderId) {
- static const TVector<TString> permissions = {"ydb.streams.write", "ydb.databases.list",
- "ydb.databases.create", "ydb.databases.connect"};
- TVector<std::pair<TString, TString>> attributes;
- if (!dbId.empty()) attributes.push_back({"database_id", dbId});
- if (!folderId.empty()) attributes.push_back({"folder_id", folderId});
- if (!attributes.empty()) {
- return {{permissions, attributes}};
- }
- return {};
-}
-
-
-
-static inline bool InternalErrorCode(NPersQueue::NErrorCode::EErrorCode errorCode) {
- switch(errorCode) {
- case NPersQueue::NErrorCode::UNKNOWN_TOPIC:
- case NPersQueue::NErrorCode::ERROR:
- case NPersQueue::NErrorCode::INITIALIZING:
- case NPersQueue::NErrorCode::OVERLOAD:
- case NPersQueue::NErrorCode::WRITE_ERROR_DISK_IS_FULL:
- return true;
- default:
- return false;
- }
- return false;
-}
-
-
-
-Ydb::StatusIds::StatusCode ConvertPersQueueInternalCodeToStatus(const NPersQueue::NErrorCode::EErrorCode code);
-void FillIssue(Ydb::Issue::IssueMessage* issue, const NPersQueue::NErrorCode::EErrorCode errorCode, const TString& errorReason);
-
-using IWriteSessionHandlerRef = TIntrusivePtr<ISessionHandler<NPersQueue::TWriteResponse>>;
-using IReadSessionHandlerRef = TIntrusivePtr<ISessionHandler<NPersQueue::TReadResponse>>;
-
-const TString& LocalDCPrefix();
-const TString& MirroredDCPrefix();
-
-constexpr ui64 MAGIC_COOKIE_VALUE = 123456789;
-
-static const TDuration CHECK_ACL_DELAY = TDuration::Minutes(5);
-
-struct TEvPQProxy {
- enum EEv {
- EvWriteInit = EventSpaceBegin(TKikimrEvents::ES_PQ_PROXY),
- EvWrite,
- EvDone,
- EvReadInit,
- EvRead,
- EvCloseSession,
- EvPartitionReady,
- EvReadResponse,
- EvCommit,
- EvCommitDone,
- EvLocked,
- EvReleasePartition,
- EvPartitionReleased,
- EvLockPartition,
- EvRestartPipe,
- EvDieCommand,
- EvPartitionStatus,
- EvAuth,
- EvReadSessionStatus,
- EvReadSessionStatusResponse,
- EvDeadlineExceeded,
- EvGetStatus,
- EvWriteDone,
- EvEnd,
- };
-
- struct TEvReadSessionStatus : public TEventPB<TEvReadSessionStatus, NKikimrPQ::TReadSessionStatus, EvReadSessionStatus> {
- };
-
- struct TEvReadSessionStatusResponse : public TEventPB<TEvReadSessionStatusResponse, NKikimrPQ::TReadSessionStatusResponse, EvReadSessionStatusResponse> {
- };
-
-
-
- struct TEvWriteInit : public NActors::TEventLocal<TEvWriteInit, EvWriteInit> {
- TEvWriteInit(const NPersQueue::TWriteRequest& req, const TString& peerName, const TString& database)
- : Request(req)
- , PeerName(peerName)
- , Database(database)
- { }
-
- NPersQueue::TWriteRequest Request;
- TString PeerName;
- TString Database;
- };
-
- struct TEvWrite : public NActors::TEventLocal<TEvWrite, EvWrite> {
- explicit TEvWrite(const NPersQueue::TWriteRequest& req)
- : Request(req)
- { }
-
- NPersQueue::TWriteRequest Request;
- };
-
- struct TEvDone : public NActors::TEventLocal<TEvDone, EvDone> {
- TEvDone()
- { }
- };
-
- struct TEvWriteDone : public NActors::TEventLocal<TEvWriteDone, EvWriteDone> {
- TEvWriteDone(ui64 size)
- : Size(size)
- { }
-
- ui64 Size;
- };
-
- struct TEvReadInit : public NActors::TEventLocal<TEvReadInit, EvReadInit> {
- TEvReadInit(const NPersQueue::TReadRequest& req, const TString& peerName, const TString& database)
- : Request(req)
- , PeerName(peerName)
- , Database(database)
- { }
-
- NPersQueue::TReadRequest Request;
- TString PeerName;
- TString Database;
- };
-
- struct TEvRead : public NActors::TEventLocal<TEvRead, EvRead> {
- explicit TEvRead(const NPersQueue::TReadRequest& req, const TString& guid = CreateGuidAsString())
- : Request(req)
- , Guid(guid)
- { }
-
- NPersQueue::TReadRequest Request;
- const TString Guid;
- };
- struct TEvCloseSession : public NActors::TEventLocal<TEvCloseSession, EvCloseSession> {
- TEvCloseSession(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode)
- : Reason(reason)
- , ErrorCode(errorCode)
- { }
-
- const TString Reason;
- NPersQueue::NErrorCode::EErrorCode ErrorCode;
- };
-
- struct TEvPartitionReady : public NActors::TEventLocal<TEvPartitionReady, EvPartitionReady> {
- TEvPartitionReady(const NPersQueue::TTopicConverterPtr& topic, const ui32 partition, const ui64 wTime, const ui64 sizeLag,
- const ui64 readOffset, const ui64 endOffset)
- : Topic(topic)
- , Partition(partition)
- , WTime(wTime)
- , SizeLag(sizeLag)
- , ReadOffset(readOffset)
- , EndOffset(endOffset)
- { }
-
- NPersQueue::TTopicConverterPtr Topic;
- ui32 Partition;
- ui64 WTime;
- ui64 SizeLag;
- ui64 ReadOffset;
- ui64 EndOffset;
- };
-
- struct TEvReadResponse : public NActors::TEventLocal<TEvReadResponse, EvReadResponse> {
- explicit TEvReadResponse(
- NPersQueue::TReadResponse&& resp,
- ui64 nextReadOffset,
- bool fromDisk,
- TDuration waitQuotaTime
- )
- : Response(std::move(resp))
- , NextReadOffset(nextReadOffset)
- , FromDisk(fromDisk)
- , WaitQuotaTime(waitQuotaTime)
- { }
-
- NPersQueue::TReadResponse Response;
- ui64 NextReadOffset;
- bool FromDisk;
- TDuration WaitQuotaTime;
- };
-
- struct TEvCommit : public NActors::TEventLocal<TEvCommit, EvCommit> {
- explicit TEvCommit(ui64 readId, ui64 offset = Max<ui64>())
- : ReadId(readId)
- , Offset(offset)
- { }
-
- ui64 ReadId;
- ui64 Offset; // Actual value for requests to concreete partitions
- };
-
- struct TEvAuth : public NActors::TEventLocal<TEvAuth, EvAuth> {
- TEvAuth(const NPersQueueCommon::TCredentials& auth)
- : Auth(auth)
- { }
-
- NPersQueueCommon::TCredentials Auth;
- };
-
- struct TEvLocked : public NActors::TEventLocal<TEvLocked, EvLocked> {
- TEvLocked(const TString& topic, ui32 partition, ui64 readOffset, ui64 commitOffset, bool verifyReadOffset, ui64 generation)
- : Topic(topic)
- , Partition(partition)
- , ReadOffset(readOffset)
- , CommitOffset(commitOffset)
- , VerifyReadOffset(verifyReadOffset)
- , Generation(generation)
- { }
-
- TString Topic;
- ui32 Partition;
- ui64 ReadOffset;
- ui64 CommitOffset;
- bool VerifyReadOffset;
- ui64 Generation;
- };
-
- struct TEvGetStatus : public NActors::TEventLocal<TEvGetStatus, EvGetStatus> {
- TEvGetStatus(const TString& topic, ui32 partition, ui64 generation)
- : Topic(topic)
- , Partition(partition)
- , Generation(generation)
- { }
-
- TString Topic;
- ui32 Partition;
- ui64 Generation;
- };
-
-
-
- struct TEvCommitDone : public NActors::TEventLocal<TEvCommitDone, EvCommitDone> {
- TEvCommitDone(ui64 readId, const NPersQueue::TTopicConverterPtr& topic, const ui32 partition)
- : ReadId(readId)
- , Topic(topic)
- , Partition(partition)
- { }
-
- ui64 ReadId;
- NPersQueue::TTopicConverterPtr Topic;
- ui32 Partition;
- };
-
- struct TEvReleasePartition : public NActors::TEventLocal<TEvReleasePartition, EvReleasePartition> {
- TEvReleasePartition()
- { }
- };
-
- struct TEvLockPartition : public NActors::TEventLocal<TEvLockPartition, EvLockPartition> {
- explicit TEvLockPartition(const ui64 readOffset, const ui64 commitOffset, bool verifyReadOffset, bool startReading)
- : ReadOffset(readOffset)
- , CommitOffset(commitOffset)
- , VerifyReadOffset(verifyReadOffset)
- , StartReading(startReading)
- { }
-
- ui64 ReadOffset;
- ui64 CommitOffset;
- bool VerifyReadOffset;
- bool StartReading;
- };
-
-
- struct TEvPartitionReleased : public NActors::TEventLocal<TEvPartitionReleased, EvPartitionReleased> {
- TEvPartitionReleased(const NPersQueue::TTopicConverterPtr& topic, const ui32 partition)
- : Topic(topic)
- , Partition(partition)
- { }
-
- NPersQueue::TTopicConverterPtr Topic;
- ui32 Partition;
- };
-
-
- struct TEvRestartPipe : public NActors::TEventLocal<TEvRestartPipe, EvRestartPipe> {
- TEvRestartPipe()
- { }
- };
-
- struct TEvDeadlineExceeded : public NActors::TEventLocal<TEvDeadlineExceeded, EvDeadlineExceeded> {
- TEvDeadlineExceeded(ui64 cookie)
- : Cookie(cookie)
- { }
-
- ui64 Cookie;
- };
-
-
- struct TEvDieCommand : public NActors::TEventLocal<TEvDieCommand, EvDieCommand> {
- TEvDieCommand(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode)
- : Reason(reason)
- , ErrorCode(errorCode)
- { }
-
- TString Reason;
- NPersQueue::NErrorCode::EErrorCode ErrorCode;
- };
-
- struct TEvPartitionStatus : public NActors::TEventLocal<TEvPartitionStatus, EvPartitionStatus> {
- TEvPartitionStatus(const NPersQueue::TTopicConverterPtr& topic, const ui32 partition, const ui64 offset,
- const ui64 endOffset, ui64 writeTimestampEstimateMs, bool init = true)
- : Topic(topic)
- , Partition(partition)
- , Offset(offset)
- , EndOffset(endOffset)
- , WriteTimestampEstimateMs(writeTimestampEstimateMs)
- , Init(init)
- { }
-
- NPersQueue::TTopicConverterPtr Topic;
- ui32 Partition;
- ui64 Offset;
- ui64 EndOffset;
- ui64 WriteTimestampEstimateMs;
- bool Init;
- };
-
-};
-
-
-
-/// WRITE ACTOR
-class TWriteSessionActor : public NActors::TActorBootstrapped<TWriteSessionActor> {
- using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
- using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
- using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
-public:
- TWriteSessionActor(IWriteSessionHandlerRef handler, const ui64 cookie, const NActors::TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TString& localDC,
- const TMaybe<TString> clientDC);
- ~TWriteSessionActor();
-
- void Bootstrap(const NActors::TActorContext& ctx);
-
- void Die(const NActors::TActorContext& ctx) override;
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::FRONT_PQ_WRITE; }
-private:
- STFUNC(StateFunc) {
- switch (ev->GetTypeRewrite()) {
- CFunc(NActors::TEvents::TSystem::Wakeup, HandleWakeup)
-
- HFunc(TEvTicketParser::TEvAuthorizeTicketResult, Handle);
-
- HFunc(TEvPQProxy::TEvDieCommand, HandlePoison)
- HFunc(TEvPQProxy::TEvWriteInit, Handle)
- HFunc(TEvPQProxy::TEvWrite, Handle)
- HFunc(TEvPQProxy::TEvDone, Handle)
- HFunc(TEvPersQueue::TEvGetPartitionIdForWriteResponse, Handle)
-
- HFunc(TEvDescribeTopicsResponse, Handle);
-
- HFunc(NPQ::TEvPartitionWriter::TEvInitResult, Handle);
- HFunc(NPQ::TEvPartitionWriter::TEvWriteAccepted, Handle);
- HFunc(NPQ::TEvPartitionWriter::TEvWriteResponse, Handle);
- HFunc(NPQ::TEvPartitionWriter::TEvDisconnected, Handle);
- HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
- HFunc(TEvTabletPipe::TEvClientConnected, Handle);
-
- HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle);
- HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle);
-
- default:
- break;
- };
- }
-
- void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TActorContext &ctx);
- void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr &ev, const TActorContext &ctx);
-
- TString CheckSupportedCodec(const ui32 codecId);
- void CheckACL(const TActorContext& ctx);
- void InitCheckACL(const TActorContext& ctx);
- void Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvWrite::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvDone::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPersQueue::TEvGetPartitionIdForWriteResponse::TPtr& ev, const NActors::TActorContext& ctx);
-
- void LogSession(const TActorContext& ctx);
-
- void InitAfterDiscovery(const TActorContext& ctx);
- void DiscoverPartition(const NActors::TActorContext& ctx);
- void SendSelectPartitionRequest(ui32 hash, const TString& topic, const NActors::TActorContext& ctx);
- void UpdatePartition(const NActors::TActorContext& ctx);
- void RequestNextPartition(const NActors::TActorContext& ctx);
- void ProceedPartition(const ui32 partition, const NActors::TActorContext& ctx);
- THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(ui32 hash, const TString& topic);
-
-
- void Handle(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx);
-
- void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx);
- void Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx);
- void Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx);
- void Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const NActors::TActorContext& ctx);
-
- void HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const NActors::TActorContext& ctx);
- void HandleWakeup(const NActors::TActorContext& ctx);
-
- void CloseSession(const TString& errorReason, const NPersQueue::NErrorCode::EErrorCode errorCode, const NActors::TActorContext& ctx);
-
- void CheckFinish(const NActors::TActorContext& ctx);
-
- void GenerateNextWriteRequest(const NActors::TActorContext& ctx);
-
- void SetupCounters();
- void SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId);
-
-
-private:
- IWriteSessionHandlerRef Handler;
-
- enum EState {
- ES_CREATED = 1,
- ES_WAIT_SCHEME_1 = 2,
- ES_WAIT_SCHEME_2 = 3,
- ES_WAIT_TABLE_REQUEST_1 = 4,
- ES_WAIT_NEXT_PARTITION = 5,
- ES_WAIT_TABLE_REQUEST_2 = 6,
- ES_WAIT_TABLE_REQUEST_3 = 7,
- ES_WAIT_WRITER_INIT = 8,
- ES_INITED = 9,
- ES_DYING = 10,
- };
-
- EState State;
- TActorId SchemeCache;
- TActorId Writer;
-
- TString PeerName;
- TString Database;
- ui64 Cookie;
-
- ui32 Partition;
- bool PartitionFound = false;
- ui32 PreferedPartition;
- TString SourceId;
- ui32 SelectReqsInflight = 0;
- ui64 MaxSrcIdAccessTime = 0;
- NPQ::NSourceIdEncoding::TEncodedSourceId EncodedSourceId;
- TString OwnerCookie;
- TString UserAgent;
-
- ui32 NumReserveBytesRequests;
-
- struct TWriteRequestBatchInfo: public TSimpleRefCount<TWriteRequestBatchInfo> {
- using TPtr = TIntrusivePtr<TWriteRequestBatchInfo>;
-
- // Source requests from user (grpc session object)
- std::deque<THolder<TEvPQProxy::TEvWrite>> UserWriteRequests;
-
- // Formed write request's size
- ui64 ByteSize = 0;
-
- // Formed write request's cookie
- ui64 Cookie = 0;
- };
-
- // Nonprocessed source client requests
- std::deque<THolder<TEvPQProxy::TEvWrite>> Writes;
-
- // Formed, but not sent, batch requests to partition actor
- std::deque<TWriteRequestBatchInfo::TPtr> FormedWrites;
-
- // Requests that is already sent to partition actor
- std::deque<TWriteRequestBatchInfo::TPtr> SentMessages;
-
- bool WritesDone;
-
- THashMap<ui32, ui64> PartitionToTablet;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
-
- NKikimr::NPQ::TMultiCounter BytesInflight;
- NKikimr::NPQ::TMultiCounter BytesInflightTotal;
-
- ui64 BytesInflight_;
- ui64 BytesInflightTotal_;
-
- bool NextRequestInited;
-
- NKikimr::NPQ::TMultiCounter SessionsCreated;
- NKikimr::NPQ::TMultiCounter SessionsActive;
- NKikimr::NPQ::TMultiCounter SessionsWithoutAuth;
-
- NKikimr::NPQ::TMultiCounter Errors;
-
- ui64 NextRequestCookie;
-
- TIntrusivePtr<NACLib::TUserToken> Token;
- NPersQueueCommon::TCredentials Auth;
- TString AuthStr;
- bool ACLCheckInProgress;
- bool FirstACLCheck;
- bool ForceACLCheck;
- bool RequestNotChecked;
- TInstant LastACLCheckTimestamp;
- TInstant LogSessionDeadline;
-
- ui64 BalancerTabletId;
- TString DatabaseId;
- TString FolderId;
- TActorId PipeToBalancer;
- TIntrusivePtr<TSecurityObject> SecurityObject;
- TPQGroupInfoPtr PQInfo;
-
- NKikimrPQClient::TDataChunk InitMeta;
- TString LocalDC;
- TString ClientDC;
- TString SelectSourceIdQuery;
- TString UpdateSourceIdQuery;
- TInstant LastSourceIdUpdate;
-
- ui64 SourceIdCreateTime = 0;
- ui32 SourceIdUpdatesInflight = 0;
-
-
- TVector<NPersQueue::TPQLabelsInfo> Aggr;
- NKikimr::NPQ::TMultiCounter SLITotal;
- NKikimr::NPQ::TMultiCounter SLIErrors;
- TInstant StartTime;
- NKikimr::NPQ::TPercentileCounter InitLatency;
- NKikimr::NPQ::TMultiCounter SLIBigLatency;
-
- THolder<NPersQueue::TTopicNamesConverterFactory> ConverterFactory;
- NPersQueue::TDiscoveryConverterPtr DiscoveryConverter;
- NPersQueue::TTopicConverterPtr FullConverter;
-
- NPersQueue::TWriteRequest::TInit InitRequest;
-};
-
-class TReadSessionActor : public TActorBootstrapped<TReadSessionActor> {
- using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
- using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
-public:
- TReadSessionActor(IReadSessionHandlerRef handler, const NPersQueue::TTopicsListController& topicsHandler, const ui64 cookie,
- const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const TMaybe<TString> clientDC);
- ~TReadSessionActor();
-
- void Bootstrap(const NActors::TActorContext& ctx);
-
- void Die(const NActors::TActorContext& ctx) override;
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::FRONT_PQ_READ; }
-
-
- struct TTopicCounters {
- NKikimr::NPQ::TMultiCounter PartitionsLocked;
- NKikimr::NPQ::TMultiCounter PartitionsReleased;
- NKikimr::NPQ::TMultiCounter PartitionsToBeReleased;
- NKikimr::NPQ::TMultiCounter PartitionsToBeLocked;
- NKikimr::NPQ::TMultiCounter PartitionsInfly;
- NKikimr::NPQ::TMultiCounter Errors;
- NKikimr::NPQ::TMultiCounter Commits;
- NKikimr::NPQ::TMultiCounter WaitsForData;
- };
-
-private:
- STFUNC(StateFunc) {
- switch (ev->GetTypeRewrite()) {
- CFunc(NActors::TEvents::TSystem::Wakeup, HandleWakeup)
-
- HFunc(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk, Handle); // form auth actor
-
- HFunc(TEvPQProxy::TEvDieCommand, HandlePoison)
- HFunc(TEvPQProxy::TEvReadInit, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvReadSessionStatus, Handle) // from read sessions info builder proxy
- HFunc(TEvPQProxy::TEvRead, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvDone, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvWriteDone, Handle) //from gRPC
- HFunc(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvCloseSession, Handle) //from partitionActor
- HFunc(TEvPQProxy::TEvCloseSession, Handle) //from partitionActor
-
- HFunc(TEvPQProxy::TEvPartitionReady, Handle) //from partitionActor
- HFunc(TEvPQProxy::TEvPartitionReleased, Handle) //from partitionActor
-
- HFunc(TEvPQProxy::TEvReadResponse, Handle) //from partitionActor
- HFunc(TEvPQProxy::TEvCommit, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvLocked, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvGetStatus, Handle) //from gRPC
- HFunc(TEvPQProxy::TEvAuth, Handle) //from gRPC
-
- HFunc(TEvPQProxy::TEvCommitDone, Handle) //from PartitionActor
- HFunc(TEvPQProxy::TEvPartitionStatus, Handle) //from partitionActor
-
- HFunc(TEvPersQueue::TEvLockPartition, Handle) //from Balancer
- HFunc(TEvPersQueue::TEvReleasePartition, Handle) //from Balancer
- HFunc(TEvPersQueue::TEvError, Handle) //from Balancer
-
- HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
- HFunc(TEvTabletPipe::TEvClientConnected, Handle);
-
- HFunc(TEvDescribeTopicsResponse, HandleDescribeTopicsResponse);
- HFunc(TEvTicketParser::TEvAuthorizeTicketResult, Handle);
-
- default:
- break;
- };
- }
-
- void Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvReadSessionStatus::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvRead::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvDone::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvWriteDone::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvCloseSession::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvCloseSession::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvPartitionReady::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvPartitionReleased::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvCommit::TPtr& ev, const NActors::TActorContext& ctx);
- void MakeCommit(const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvLocked::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvAuth::TPtr& ev, const NActors::TActorContext& ctx);
- void ProcessAuth(const NPersQueueCommon::TCredentials& auth);
- void Handle(TEvPQProxy::TEvCommitDone::TPtr& ev, const NActors::TActorContext& ctx);
- void AnswerForCommitsIfCan(const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const NActors::TActorContext& ctx);
-
- void Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPersQueue::TEvError::TPtr& ev, const NActors::TActorContext& ctx);
-
- void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const NActors::TActorContext& ctx);
- [[nodiscard]] bool ProcessBalancerDead(const ui64 tabletId, const NActors::TActorContext& ctx); // returns false if actor died
-
- void HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const NActors::TActorContext& ctx);
- void HandleWakeup(const NActors::TActorContext& ctx);
- void Handle(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const NActors::TActorContext& ctx);
-
- void CloseSession(const TString& errorReason, const NPersQueue::NErrorCode::EErrorCode errorCode,
- const NActors::TActorContext& ctx);
-
- void Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx);
- void HandleDescribeTopicsResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx);
-
- void SendAuthRequest(const TActorContext& ctx);
- void CreateInitAndAuthActor(const TActorContext& ctx);
-
- void SetupCounters();
- void SetupTopicCounters(const NPersQueue::TTopicConverterPtr& topic);
- void SetupTopicCounters(const NPersQueue::TTopicConverterPtr& topic, const TString& cloudId, const TString& dbId,
- const TString& folderId);
-
- [[nodiscard]] bool ProcessReads(const NActors::TActorContext& ctx); // returns false if actor died
- struct TFormedReadResponse;
- [[nodiscard]] bool ProcessAnswer(const NActors::TActorContext& ctx, TIntrusivePtr<TFormedReadResponse> formedResponse); // returns false if actor died
-
- void RegisterSessions(const NActors::TActorContext& ctx);
- void RegisterSession(const TActorId& pipe, const TString& topic, const TActorContext& ctx);
-
- struct TPartitionActorInfo;
- void DropPartitionIfNeeded(THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator it, const TActorContext& ctx);
-
- bool ActualPartitionActor(const TActorId& part);
- [[nodiscard]] bool ProcessReleasePartition(const THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator& it,
- bool kill, bool couldBeReads, const TActorContext& ctx); // returns false if actor died
- void InformBalancerAboutRelease(const THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator& it, const TActorContext& ctx);
-
- // returns false if check failed.
- bool CheckAndUpdateReadSettings(const NPersQueue::TReadRequest::TRead& readRequest);
-
- static ui32 NormalizeMaxReadMessagesCount(ui32 sourceValue);
- static ui32 NormalizeMaxReadSize(ui32 sourceValue);
- static ui32 NormalizeMaxReadPartitionsCount(ui32 sourceValue);
-
- static bool RemoveEmptyMessages(NPersQueue::TReadResponse::TBatchedData& data); // returns true if there are nonempty messages
-
-private:
- IReadSessionHandlerRef Handler;
-
- const TInstant StartTimestamp;
-
- TActorId PqMetaCache;
- TActorId NewSchemeCache;
-
- TActorId AuthInitActor;
- bool AuthInflight;
-
- TString InternalClientId;
- TString ExternalClientId;
- const TString ClientDC;
- TString ClientPath;
- TString Session;
- TString PeerName;
- TString Database;
-
- bool ClientsideLocksAllowed;
- bool BalanceRightNow;
- bool CommitsDisabled;
- bool BalancersInitStarted;
-
- bool InitDone;
-
- ui32 ProtocolVersion; // from NPersQueue::TReadRequest::EProtocolVersion
- // Read settings.
- // Can be initialized during Init request (new preferable way)
- // or during read request (old way that will be removed in future).
- // These settings can't be changed (in that case server closes session).
- ui32 MaxReadMessagesCount;
- ui32 MaxReadSize;
- ui32 MaxReadPartitionsCount;
- ui32 MaxTimeLagMs;
- ui64 ReadTimestampMs;
- bool ReadSettingsInited;
-
- NPersQueueCommon::TCredentials Auth;
- TString AuthStr;
- TIntrusivePtr<NACLib::TUserToken> Token;
- bool ForceACLCheck;
- bool RequestNotChecked;
- TInstant LastACLCheckTimestamp;
-
- struct TPartitionActorInfo {
- TActorId Actor;
- std::deque<ui64> Commits;
- bool Reading;
- bool Releasing;
- bool Released;
- ui64 LockGeneration;
- bool LockSent;
- NPersQueue::TTopicConverterPtr Converter;
-
- TPartitionActorInfo(const TActorId& actor, ui64 generation, const NPersQueue::TTopicConverterPtr& topic)
- : Actor(actor)
- , Reading(false)
- , Releasing(false)
- , Released(false)
- , LockGeneration(generation)
- , LockSent(false)
- , Converter(topic)
- {}
- };
-
-
- THashSet<TActorId> ActualPartitionActors;
- THashMap<std::pair<TString, ui32>, TPartitionActorInfo> Partitions; //topic[ClientSideName!]:partition -> info
-
- THashMap<TString, NPersQueue::TTopicConverterPtr> FullPathToConverter; // PrimaryFullPath -> Converter, for balancer replies matching
- THashMap<TString, TTopicHolder> Topics; // PrimaryName ->topic info
-
- TVector<ui32> Groups;
- bool ReadOnlyLocal;
-
- struct TPartitionInfo {
- NPersQueue::TTopicConverterPtr Topic;
- ui32 Partition;
- ui64 WTime;
- ui64 SizeLag;
- ui64 MsgLag;
- TActorId Actor;
- bool operator < (const TPartitionInfo& rhs) const {
- return std::tie(WTime, Topic, Partition, Actor) < std::tie(rhs.WTime, rhs.Topic, rhs.Partition, rhs.Actor);
- }
- };
-
- TSet<TPartitionInfo> AvailablePartitions;
-
- struct TOffsetsInfo {
- struct TPartitionOffsetInfo {
- TPartitionOffsetInfo(const TActorId& sender, const TString& topic, ui32 partition, ui64 offset)
- : Sender(sender)
- , Topic(topic)
- , Partition(partition)
- , Offset(offset)
- {
- }
-
- TActorId Sender;
- TString Topic;
- ui32 Partition;
- ui64 Offset;
- };
-
- // find by read id
- bool operator<(ui64 readId) const {
- return ReadId < readId;
- }
-
- friend bool operator<(ui64 readId, const TOffsetsInfo& info) {
- return readId < info.ReadId;
- }
-
- ui64 ReadId = 0;
- std::vector<TPartitionOffsetInfo> PartitionOffsets;
- };
-
- std::deque<TOffsetsInfo> Offsets; // Sequential read id -> offsets
-
- struct TFormedReadResponse: public TSimpleRefCount<TFormedReadResponse> {
- using TPtr = TIntrusivePtr<TFormedReadResponse>;
-
- TFormedReadResponse(const TString& guid, const TInstant start)
- : Guid(guid)
- , Start(start)
- , FromDisk(false)
- {
- }
-
- NPersQueue::TReadResponse Response;
- ui32 RequestsInfly = 0;
- i64 ByteSize = 0;
-
- ui64 RequestedBytes = 0;
-
- //returns byteSize diff
- i64 ApplyResponse(NPersQueue::TReadResponse&& resp);
-
- TVector<NPersQueue::TReadResponse> ControlMessages;
-
- THashSet<TActorId> PartitionsTookPartInRead;
- TSet<TPartitionInfo> PartitionsBecameAvailable; // Partitions that became available during this read request execution.
- // These partitions are bringed back to AvailablePartitions after reply to this read request.
- TOffsetsInfo Offsets; // Offsets without assigned read id.
-
- const TString Guid;
- TInstant Start;
- bool FromDisk;
- TDuration WaitQuotaTime;
- };
-
- THashMap<TActorId, TFormedReadResponse::TPtr> PartitionToReadResponse; // Partition actor -> TFormedReadResponse answer that has this partition.
- // PartitionsTookPartInRead in formed read response contain this actor id.
-
- ui64 ReadIdToResponse;
- ui64 ReadIdCommitted;
- TSet<ui64> NextCommits;
- TInstant LastCommitTimestamp;
- TDuration CommitInterval;
- ui32 CommitsInfly;
-
- std::deque<THolder<TEvPQProxy::TEvRead>> Reads;
-
- ui64 Cookie;
-
- struct TCommitInfo {
- ui64 StartReadId;
- ui32 Partitions;
- TInstant StartTime;
- };
-
- TMap<ui64, TCommitInfo> Commits; //readid->TCommitInfo
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
-
- NMonitoring::TDynamicCounters::TCounterPtr SessionsCreated;
- NMonitoring::TDynamicCounters::TCounterPtr SessionsActive;
- NMonitoring::TDynamicCounters::TCounterPtr SessionsWithoutAuth;
- NMonitoring::TDynamicCounters::TCounterPtr SessionsWithOldBatchingVersion; // LOGBROKER-3173
-
- NMonitoring::TDynamicCounters::TCounterPtr Errors;
- NMonitoring::TDynamicCounters::TCounterPtr PipeReconnects;
- NMonitoring::TDynamicCounters::TCounterPtr BytesInflight;
- ui64 BytesInflight_;
- ui64 RequestedBytes;
- ui32 ReadsInfly;
-
- NKikimr::NPQ::TPercentileCounter PartsPerSession;
-
- THashMap<TString, TTopicCounters> TopicCounters;
- THashMap<TString, ui32> NumPartitionsFromTopic;
-
- TVector<NPersQueue::TPQLabelsInfo> Aggr;
- NKikimr::NPQ::TMultiCounter SLITotal;
- NKikimr::NPQ::TMultiCounter SLIErrors;
- TInstant StartTime;
- NKikimr::NPQ::TPercentileCounter InitLatency;
- NKikimr::NPQ::TPercentileCounter CommitLatency;
- NKikimr::NPQ::TMultiCounter SLIBigLatency;
-
- NKikimr::NPQ::TPercentileCounter ReadLatency;
- NKikimr::NPQ::TPercentileCounter ReadLatencyFromDisk;
- NKikimr::NPQ::TMultiCounter SLIBigReadLatency;
- NKikimr::NPQ::TMultiCounter ReadsTotal;
-
- NPersQueue::TTopicsListController TopicsHandler;
- NPersQueue::TTopicsToConverter TopicsList;
-};
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.cpp b/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.cpp
deleted file mode 100644
index 357c535aca..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-#include "grpc_pq_clusters_updater_actor.h"
-
-#include <ydb/core/base/appdata.h>
-#include <ydb/core/persqueue/pq_database.h>
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-static const int CLUSTERS_UPDATER_TIMEOUT_ON_ERROR = 1;
-
-
-TClustersUpdater::TClustersUpdater(IPQClustersUpdaterCallback* callback)
- : Callback(callback)
- {};
-
-void TClustersUpdater::Bootstrap(const NActors::TActorContext& ctx) {
- ctx.Send(ctx.SelfID, new TEvPQClustersUpdater::TEvUpdateClusters());
- ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
-
- Become(&TThis::StateFunc);
-}
-
-void TClustersUpdater::Handle(TEvPQClustersUpdater::TEvUpdateClusters::TPtr&, const TActorContext &ctx) {
- auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
- req->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- req->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
- req->Record.MutableRequest()->SetKeepSession(false);
- req->Record.MutableRequest()->SetQuery("--!syntax_v1\nSELECT `name`, `local`, `enabled` FROM `" + AppData(ctx)->PQConfig.GetRoot() + "/Config/V2/Cluster`;");
- req->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig));
- req->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
- req->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), req.Release());
-}
-
-void TClustersUpdater::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev, const TActorContext&) {
-
- Callback->NetClassifierUpdated(ev->Get()->Classifier);
-}
-
-
-
-
-void TClustersUpdater::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TActorContext &ctx) {
- auto& record = ev->Get()->Record.GetRef();
-
- if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) {
- auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0);
- bool local = false;
- TVector<TString> clusters;
- for (size_t i = 0; i < t.ListSize(); ++i) {
- TString dc = t.GetList(i).GetStruct(0).GetOptional().GetText();
- local = t.GetList(i).GetStruct(1).GetOptional().GetBool();
- clusters.push_back(dc);
- if (local) {
- bool enabled = t.GetList(i).GetStruct(2).GetOptional().GetBool();
- Y_VERIFY(LocalCluster.empty() || LocalCluster == dc);
- bool changed = LocalCluster != dc || Enabled != enabled;
- if (changed) {
- LocalCluster = dc;
- Enabled = enabled;
- Callback->CheckClusterChange(LocalCluster, Enabled);
- }
- }
- }
- if (Clusters != clusters) {
- Clusters = clusters;
- Callback->CheckClustersListChange(Clusters);
- }
- ctx.Schedule(TDuration::Seconds(AppData(ctx)->PQConfig.GetClustersUpdateTimeoutSec()), new TEvPQClustersUpdater::TEvUpdateClusters());
- } else {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "can't update clusters " << record);
- ctx.Schedule(TDuration::Seconds(CLUSTERS_UPDATER_TIMEOUT_ON_ERROR), new TEvPQClustersUpdater::TEvUpdateClusters());
- }
-}
-
-
-void TClustersUpdater::Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr &ev, const TActorContext &ctx) {
- auto& record = ev->Get()->Record;
-
- LOG_ERROR_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "can't update clusters " << record);
- ctx.Schedule(TDuration::Seconds(CLUSTERS_UPDATER_TIMEOUT_ON_ERROR), new TEvPQClustersUpdater::TEvUpdateClusters());
-}
-
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.h b/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.h
deleted file mode 100644
index 6f1b6ade76..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_clusters_updater_actor.h
+++ /dev/null
@@ -1,77 +0,0 @@
-#pragma once
-
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_local.h>
-#include <library/cpp/actors/core/hfunc.h>
-
-#include <ydb/core/base/events.h>
-#include <ydb/core/kqp/kqp.h>
-#include <ydb/core/mind/address_classification/net_classifier.h>
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-struct TEvPQClustersUpdater {
- enum EEv {
- EvUpdateClusters = EventSpaceBegin(TKikimrEvents::ES_PQ_CLUSTERS_UPDATER),
- EvEnd,
- };
-
- struct TEvUpdateClusters : public NActors::TEventLocal<TEvUpdateClusters, EvUpdateClusters> {
- TEvUpdateClusters()
- {}
- };
-};
-
-class IPQClustersUpdaterCallback {
-public:
- virtual ~IPQClustersUpdaterCallback() = default;
- virtual void CheckClusterChange(const TString& localCluster, const bool enabled)
- {
- Y_UNUSED(localCluster);
- Y_UNUSED(enabled);
- }
-
- virtual void CheckClustersListChange(const TVector<TString>& clusters)
- {
- Y_UNUSED(clusters);
- }
-
- virtual void NetClassifierUpdated(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier) {
- Y_UNUSED(classifier);
- }
-};
-
-class TClustersUpdater : public NActors::TActorBootstrapped<TClustersUpdater> {
-public:
- TClustersUpdater(IPQClustersUpdaterCallback* callback);
-
- void Bootstrap(const NActors::TActorContext& ctx);
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::FRONT_PQ_WRITE; } // FIXME
-
-private:
- IPQClustersUpdaterCallback* Callback;
- TString LocalCluster;
- TVector<TString> Clusters;
- bool Enabled = false;
-
- STFUNC(StateFunc) {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvPQClustersUpdater::TEvUpdateClusters, Handle);
- HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle);
- HFunc(NKqp::TEvKqp::TEvProcessResponse, Handle);
- HFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, Handle);
- }
- }
-
- void Handle(TEvPQClustersUpdater::TEvUpdateClusters::TPtr &ev, const TActorContext &ctx);
- void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TActorContext &ctx);
- void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr &ev, const TActorContext &ctx);
- void Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev, const TActorContext& ctx);
-
-};
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.cpp b/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.cpp
deleted file mode 100644
index 373b74bcf2..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.cpp
+++ /dev/null
@@ -1,268 +0,0 @@
-#include "grpc_pq_read.h"
-#include "grpc_pq_actor.h"
-#include "grpc_pq_session.h"
-#include "ydb/core/client/server/grpc_proxy_status.h"
-
-#include <ydb/core/grpc_services/grpc_helper.h>
-#include <ydb/core/tx/scheme_board/cache.h>
-
-using namespace NActors;
-using namespace NKikimrClient;
-
-using grpc::Status;
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-///////////////////////////////////////////////////////////////////////////////
-
-using namespace NPersQueue;
-
-void TPQReadService::TSession::OnCreated() {
- // Start waiting for new session.
- Proxy->WaitReadSession();
- if (Proxy->TooMuchSessions()) {
- ReplyWithError("proxy overloaded", NPersQueue::NErrorCode::OVERLOAD);
- return;
- }
- // Create actor for current session.
- auto clusters = Proxy->GetClusters();
- auto localCluster = Proxy->GetLocalCluster();
- if (NeedDiscoverClusters && (clusters.empty() || localCluster.empty())) {
- //TODO: inc sli errors counter
- ReplyWithError("clusters list or local cluster is empty", NPersQueue::NErrorCode::INITIALIZING);
- return;
-
- }
- if (!TopicConverterFactory->GetLocalCluster().empty()) {
- TopicConverterFactory->SetLocalCluster(localCluster);
- }
- auto topicsHandler = std::make_unique<NPersQueue::TTopicsListController>(
- TopicConverterFactory, clusters
- );
-
- CreateActor(std::move(topicsHandler));
- ReadyForNextRead();
-}
-
-void TPQReadService::TSession::OnRead(const NPersQueue::TReadRequest& request) {
- switch (request.GetRequestCase()) {
- case TReadRequest::kInit: {
- SendEvent(new TEvPQProxy::TEvReadInit(request, GetPeerName(), GetDatabase()));
- break;
- }
- case TReadRequest::kRead: {
- SendEvent(new TEvPQProxy::TEvRead(request));
- break;
- }
- case TReadRequest::kStatus: {
- Y_VERIFY(ActorId);
- const auto& req = request.GetStatus();
- const TString& topic = req.GetTopic();
- const ui32 partition = req.GetPartition();
- const ui64 generation = req.GetGeneration();
- SendEvent(new TEvPQProxy::TEvGetStatus(topic, partition, generation));
- ReadyForNextRead();
- break;
- }
- case TReadRequest::kStartRead: {
- Y_VERIFY(ActorId);
- const auto& req = request.GetStartRead();
- const TString& topic = req.GetTopic();
- const ui32 partition = req.GetPartition();
- const ui64 readOffset = req.GetReadOffset();
- const ui64 commitOffset = req.GetCommitOffset();
- const bool verifyReadOffset = req.GetVerifyReadOffset();
- const ui64 generation = req.GetGeneration();
-
- if (request.GetCredentials().GetCredentialsCase() != NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- SendEvent(new TEvPQProxy::TEvAuth(request.GetCredentials()));
- }
- SendEvent(new TEvPQProxy::TEvLocked(topic, partition, readOffset, commitOffset, verifyReadOffset, generation));
- ReadyForNextRead();
- break;
- }
- case TReadRequest::kCommit: {
- Y_VERIFY(ActorId);
- const auto& req = request.GetCommit();
-
- if (request.GetCredentials().GetCredentialsCase() != NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- SendEvent(new TEvPQProxy::TEvAuth(request.GetCredentials()));
- }
-
- // Empty cookies list will lead to no effect.
- for (ui32 i = 0; i < req.CookieSize(); ++i) {
- SendEvent(new TEvPQProxy::TEvCommit(req.GetCookie(i)));
- }
-
- ReadyForNextRead();
- break;
- }
-
- default: {
- SendEvent(new TEvPQProxy::TEvCloseSession("unsupported request", NPersQueue::NErrorCode::BAD_REQUEST));
- break;
- }
- }
-}
-
-void TPQReadService::TSession::OnDone() {
- SendEvent(new TEvPQProxy::TEvDone());
-}
-
-void TPQReadService::TSession::OnWriteDone(ui64 size) {
- SendEvent(new TEvPQProxy::TEvWriteDone(size));
-}
-
-void TPQReadService::TSession::DestroyStream(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) {
- // Send poison pill to the actor(if it is alive)
- SendEvent(new TEvPQProxy::TEvDieCommand("read-session " + ToString<ui64>(Cookie) + ": " + reason, errorCode));
- // Remove reference to session from "cookie -> session" map.
- Proxy->ReleaseSession(Cookie);
-}
-
-bool TPQReadService::TSession::IsShuttingDown() const {
- return Proxy->IsShuttingDown();
-}
-
-TPQReadService::TSession::TSession(std::shared_ptr<TPQReadService> proxy,
- grpc::ServerCompletionQueue* cq, ui64 cookie, const TActorId& schemeCache, const TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, bool needDiscoverClusters,
- const NPersQueue::TConverterFactoryPtr& converterFactory)
- : ISession(cq)
- , Proxy(proxy)
- , Cookie(cookie)
- , ActorId()
- , SchemeCache(schemeCache)
- , NewSchemeCache(newSchemeCache)
- , Counters(counters)
- , NeedDiscoverClusters(needDiscoverClusters)
- , TopicConverterFactory(converterFactory)
-{
-}
-
-void TPQReadService::TSession::Start() {
- if (!Proxy->IsShuttingDown()) {
- Proxy->RequestSession(&Context, &Stream, CQ, CQ, new TRequestCreated(this));
- }
-}
-
-void TPQReadService::TSession::SendEvent(IEventBase* ev) {
- Proxy->ActorSystem->Send(ActorId, ev);
-}
-
-void TPQReadService::TSession::CreateActor(std::unique_ptr<NPersQueue::TTopicsListController>&& topicsHandler) {
- auto classifier = Proxy->GetClassifier();
-
- ActorId = Proxy->ActorSystem->Register(
- new TReadSessionActor(this, *topicsHandler, Cookie, SchemeCache, NewSchemeCache, Counters,
- classifier ? classifier->ClassifyAddress(GetPeerName())
- : "unknown"));
-}
-
-
-
-ui64 TPQReadService::TSession::GetCookie() const {
- return Cookie;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-
-TPQReadService::TPQReadService(NKikimr::NGRpcService::TGRpcPersQueueService* service, grpc::ServerCompletionQueue* cq,
- NActors::TActorSystem* as, const TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const ui32 maxSessions)
- : Service(service)
- , CQ(cq)
- , ActorSystem(as)
- , SchemeCache(schemeCache)
- , Counters(counters)
- , MaxSessions(maxSessions)
-{
- auto appData = ActorSystem->AppData<TAppData>();
- auto cacheCounters = GetServiceCounters(counters, "pqproxy|schemecache");
- auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters);
- NewSchemeCache = ActorSystem->Register(CreateSchemeBoardSchemeCache(cacheConfig.Get()));
- // ToDo[migration]: Other conditions;
- NeedDiscoverClusters = !ActorSystem->AppData<TAppData>()->PQConfig.GetTopicsAreFirstClassCitizen();
- TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>(
- ActorSystem->AppData<TAppData>()->PQConfig, ""
- );
-
- if (NeedDiscoverClusters) {
- ActorSystem->Register(new TClustersUpdater(this));
- }
-}
-
-
-ui64 TPQReadService::NextCookie() {
- return AtomicIncrement(LastCookie);
-}
-
-
-void TPQReadService::ReleaseSession(ui64 cookie) {
- auto g(Guard(Lock));
- bool erased = Sessions.erase(cookie);
- if (erased)
- ActorSystem->Send(MakeGRpcProxyStatusID(ActorSystem->NodeId), new TEvGRpcProxyStatus::TEvUpdateStatus(0,0,-1,0));
-
-}
-
-void TPQReadService::CheckClusterChange(const TString& localCluster, const bool) {
- auto g(Guard(Lock));
- LocalCluster = localCluster;
- TopicConverterFactory->SetLocalCluster(localCluster);
-}
-
-void TPQReadService::NetClassifierUpdated(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier) {
- auto g(Guard(Lock));
- if (!DatacenterClassifier) {
- for (auto it = Sessions.begin(); it != Sessions.end();) {
- auto jt = it++;
- jt->second->DestroyStream("datacenter classifier initialized, restart session please", NPersQueue::NErrorCode::INITIALIZING);
- }
- }
-
- DatacenterClassifier = classifier;
-}
-
-
-void TPQReadService::CheckClustersListChange(const TVector<TString> &clusters) {
- auto g(Guard(Lock));
- Clusters = clusters;
-}
-
-void TPQReadService::SetupIncomingRequests() {
- WaitReadSession();
-}
-
-
-void TPQReadService::WaitReadSession() {
-
- const ui64 cookie = NextCookie();
-
- ActorSystem->Send(MakeGRpcProxyStatusID(ActorSystem->NodeId), new TEvGRpcProxyStatus::TEvUpdateStatus(0,0,1,0));
-
- TSessionRef session(new TSession(shared_from_this(), CQ, cookie, SchemeCache, NewSchemeCache, Counters,
- NeedDiscoverClusters, TopicConverterFactory));
-
- {
- auto g(Guard(Lock));
- Sessions.insert(std::make_pair(cookie, session));
- }
-
- session->Start();
-}
-
-
-
-bool TPQReadService::TooMuchSessions() {
- auto g(Guard(Lock));
- return Sessions.size() >= MaxSessions;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.h b/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.h
deleted file mode 100644
index 9fbc177e6f..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read.h
+++ /dev/null
@@ -1,146 +0,0 @@
-#pragma once
-
-#include "persqueue.h"
-#include "grpc_pq_clusters_updater_actor.h"
-#include "grpc_pq_session.h"
-
-#include <ydb/core/client/server/grpc_base.h>
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-
-#include <library/cpp/grpc/server/grpc_request.h>
-#include <library/cpp/actors/core/actorsystem.h>
-
-#include <util/generic/hash.h>
-#include <util/system/mutex.h>
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-class TPQReadService : public IPQClustersUpdaterCallback, public std::enable_shared_from_this<TPQReadService> {
- class TSession
- : public ISession<NPersQueue::TReadRequest, NPersQueue::TReadResponse>
- {
-
- public:
- void OnCreated() override;
- void OnRead(const NPersQueue::TReadRequest& request) override;
- void OnDone() override;
- void OnWriteDone(ui64 size) override;
- void DestroyStream(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) override;
- bool IsShuttingDown() const override;
- TSession(std::shared_ptr<TPQReadService> proxy,
- grpc::ServerCompletionQueue* cq, ui64 cookie, const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, bool needDiscoverClusters,
- const NPersQueue::TConverterFactoryPtr& converterFactory);
- void Start() override;
- void SendEvent(NActors::IEventBase* ev);
-
- private:
- void CreateActor(std::unique_ptr<NPersQueue::TTopicsListController>&& topicsHandler);
- ui64 GetCookie() const;
-
- private:
- std::shared_ptr<TPQReadService> Proxy;
- const ui64 Cookie;
-
- NActors::TActorId ActorId;
-
- const NActors::TActorId SchemeCache;
- const NActors::TActorId NewSchemeCache;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- bool NeedDiscoverClusters;
-
- NPersQueue::TConverterFactoryPtr TopicConverterFactory;
-
- };
-
- using TSessionRef = TIntrusivePtr<TSession>;
-
-public:
-
- TPQReadService(NGRpcService::TGRpcPersQueueService* service,
- grpc::ServerCompletionQueue* cq,
- NActors::TActorSystem* as, const NActors::TActorId& schemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const ui32 maxSessions);
-
- virtual ~TPQReadService()
- {}
-
- void RequestSession(::grpc::ServerContext* context, ::grpc::ServerAsyncReaderWriter< ::NPersQueue::TReadResponse, ::NPersQueue::TReadRequest>* stream,
- ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag)
- {
- Service->GetService()->RequestReadSession(context, stream, new_call_cq, notification_cq, tag);
- }
-
- void SetupIncomingRequests();
-
- void StopService() {
- AtomicSet(ShuttingDown_, 1);
- }
-
- bool IsShuttingDown() const {
- return AtomicGet(ShuttingDown_);
- }
-
- TVector<TString> GetClusters() const {
- auto g(Guard(Lock));
- return Clusters;
- }
- TString GetLocalCluster() const {
- auto g(Guard(Lock));
- return LocalCluster;
- }
-
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr GetClassifier() const {
- auto g(Guard(Lock));
- return DatacenterClassifier;
- }
-
-private:
- ui64 NextCookie();
-
- void CheckClustersListChange(const TVector<TString>& clusters) override;
- void CheckClusterChange(const TString& localCluster, const bool enabled) override;
- void NetClassifierUpdated(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier) override;
- void UpdateTopicsHandler();
- //! Unregistry session object.
- void ReleaseSession(ui64 cookie);
-
- //! Start listening for incoming connections.
- void WaitReadSession();
-
- bool TooMuchSessions();
-
-private:
- NKikimr::NGRpcService::TGRpcPersQueueService* Service;
-
- grpc::ServerContext Context;
- grpc::ServerCompletionQueue* CQ;
- NActors::TActorSystem* ActorSystem;
- NActors::TActorId SchemeCache;
- NActors::TActorId NewSchemeCache;
-
- TAtomic LastCookie = 0;
- TMutex Lock;
- THashMap<ui64, TSessionRef> Sessions;
-
- TVector<TString> Clusters;
- TString LocalCluster;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
-
- ui32 MaxSessions;
-
- TAtomic ShuttingDown_ = 0;
-
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
-
- bool NeedDiscoverClusters;
- NPersQueue::TConverterFactoryPtr TopicConverterFactory;
- std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler;
-};
-
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read_actor.cpp b/kikimr/yndx/grpc_services/persqueue/grpc_pq_read_actor.cpp
deleted file mode 100644
index e9e8798713..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_read_actor.cpp
+++ /dev/null
@@ -1,2585 +0,0 @@
-#include "grpc_pq_actor.h"
-
-#include <ydb/core/base/path.h>
-#include <ydb/core/client/server/msgbus_server_persqueue.h>
-#include <ydb/core/protos/services.pb.h>
-#include <ydb/core/persqueue/percentile_counter.h>
-#include <ydb/core/persqueue/pq_database.h>
-#include <ydb/core/persqueue/write_meta.h>
-#include <ydb/core/persqueue/writer/source_id_encoding.h>
-#include <ydb/library/persqueue/topic_parser/type_definitions.h>
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <ydb/library/persqueue/topic_parser/counters.h>
-#include <kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h>
-
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/actors/interconnect/interconnect.h>
-#include <library/cpp/protobuf/util/repeated_field_utils.h>
-
-#include <util/string/strip.h>
-#include <util/charset/utf8.h>
-
-#include <algorithm>
-
-using namespace NActors;
-using namespace NKikimrClient;
-
-namespace NKikimr {
-
-using namespace NMsgBusProxy;
-
-namespace NGRpcProxy {
-
-using namespace NPersQueue;
-using namespace NSchemeCache;
-#define PQ_LOG_PREFIX "session cookie " << Cookie << " client " << InternalClientId << " session " << Session
-
-
-//11 tries = 10,23 seconds, then each try for 5 seconds , so 21 retries will take near 1 min
-static const NTabletPipe::TClientRetryPolicy RetryPolicyForPipes = {
- .RetryLimitCount = 21,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::Seconds(5),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
-};
-
-static const ui64 MAX_INFLY_BYTES = 25_MB;
-static const ui32 MAX_INFLY_READS = 10;
-
-static const TDuration READ_TIMEOUT_DURATION = TDuration::Seconds(1);
-
-static const TDuration WAIT_DATA = TDuration::Seconds(10);
-static const TDuration PREWAIT_DATA = TDuration::Seconds(9);
-static const TDuration WAIT_DELTA = TDuration::MilliSeconds(500);
-
-static const ui64 INIT_COOKIE = Max<ui64>(); //some identifier
-
-static const ui32 MAX_PIPE_RESTARTS = 100; //after 100 restarts without progress kill session
-static const ui32 RESTART_PIPE_DELAY_MS = 100;
-
-static const ui64 MAX_READ_SIZE = 100 << 20; //100mb;
-
-static const TDuration DEFAULT_COMMIT_RATE = TDuration::Seconds(1); //1 second;
-static const ui32 MAX_COMMITS_INFLY = 3;
-
-static const double LAG_GROW_MULTIPLIER = 1.2; //assume that 20% more data arrived to partitions
-
-
-//TODO: add here tracking of bytes in/out
-
-#define LOG_PROTO(FieldName) \
- if (proto.Has##FieldName()) { \
- res << " " << Y_STRINGIZE(FieldName) << " { " << proto.Get##FieldName().ShortDebugString() << " }"; \
- }
-
-#define LOG_FIELD(proto, FieldName) \
- if (proto.Has##FieldName()) { \
- res << " " << Y_STRINGIZE(FieldName) << ": " << proto.Get##FieldName(); \
- }
-
-TString PartitionResponseToLog(const NKikimrClient::TPersQueuePartitionResponse& proto) {
- if (!proto.HasCmdReadResult()) {
- return proto.ShortDebugString();
- }
- TStringBuilder res;
- res << "{";
-
-
- if (proto.CmdWriteResultSize() > 0) {
- res << " CmdWriteResult {";
- for (const auto& writeRes : proto.GetCmdWriteResult()) {
- res << " { " << writeRes.ShortDebugString() << " }";
- }
- res << " }";
- }
-
- LOG_PROTO(CmdGetMaxSeqNoResult);
- LOG_PROTO(CmdGetClientOffsetResult);
- LOG_PROTO(CmdGetOwnershipResult);
-
-
- if (proto.HasCmdReadResult()) {
- const auto& readRes = proto.GetCmdReadResult();
- res << " CmdReadResult {";
- LOG_FIELD(readRes, MaxOffset);
- LOG_FIELD(readRes, BlobsFromDisk);
- LOG_FIELD(readRes, BlobsFromCache);
- //LOG_FIELD(readRes, ErrorCode);
- LOG_FIELD(readRes, ErrorReason);
- LOG_FIELD(readRes, BlobsCachedSize);
- LOG_FIELD(readRes, SizeLag);
- LOG_FIELD(readRes, RealReadOffset);
- if (readRes.ResultSize() > 0) {
- res << " Result {";
- for (const auto &tRes: readRes.GetResult()) {
- res << " {";
- LOG_FIELD(tRes, Offset);
- LOG_FIELD(tRes, SeqNo);
- LOG_FIELD(tRes, PartNo);
- LOG_FIELD(tRes, TotalParts);
- LOG_FIELD(tRes, TotalSize);
- LOG_FIELD(tRes, WriteTimestampMS);
- LOG_FIELD(tRes, CreateTimestampMS);
- LOG_FIELD(tRes, UncompressedSize);
- LOG_FIELD(tRes, PartitionKey);
- res << " }";
- }
- res << " }";
- }
- res << " }";
- }
- res << " }";
- return res;
-}
-#undef LOG_PROTO
-#undef LOG_FIELD
-
-class TPartitionActor : public NActors::TActorBootstrapped<TPartitionActor> {
-public:
- TPartitionActor(const TActorId& parentId, const TString& clientId, const ui64 cookie, const TString& session, const ui32 generation,
- const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const ui32 partition, const ui64 tabletID,
- const TReadSessionActor::TTopicCounters& counters, const TString& clientDC);
- ~TPartitionActor();
-
- void Bootstrap(const NActors::TActorContext& ctx);
- void Die(const NActors::TActorContext& ctx) override;
-
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::FRONT_PQ_PARTITION; }
-private:
- STFUNC(StateFunc) {
- switch (ev->GetTypeRewrite()) {
- CFunc(NActors::TEvents::TSystem::Wakeup, HandleWakeup)
- HFunc(TEvPQProxy::TEvDeadlineExceeded, Handle)
-
- HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison)
- HFunc(TEvPQProxy::TEvRead, Handle)
- HFunc(TEvPQProxy::TEvCommit, Handle)
- HFunc(TEvPQProxy::TEvReleasePartition, Handle)
- HFunc(TEvPQProxy::TEvLockPartition, Handle)
- HFunc(TEvPQProxy::TEvGetStatus, Handle)
- HFunc(TEvPQProxy::TEvRestartPipe, Handle)
-
- HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
- HFunc(TEvTabletPipe::TEvClientConnected, Handle);
- HFunc(TEvPersQueue::TEvResponse, Handle);
- HFunc(TEvPersQueue::TEvHasDataInfoResponse, Handle);
- default:
- break;
- };
- }
-
-
- void Handle(TEvPQProxy::TEvReleasePartition::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const NActors::TActorContext& ctx);
-
- void Handle(TEvPQProxy::TEvDeadlineExceeded::TPtr& ev, const NActors::TActorContext& ctx);
-
- void Handle(TEvPQProxy::TEvRead::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvCommit::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const NActors::TActorContext& ctx);
-
- void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, const NActors::TActorContext& ctx);
-
- void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr& ev, const NActors::TActorContext& ctx);
- void HandleWakeup(const NActors::TActorContext& ctx);
-
- void CheckRelease(const NActors::TActorContext& ctx);
- void InitLockPartition(const NActors::TActorContext& ctx);
- void InitStartReading(const NActors::TActorContext& ctx);
-
- void RestartPipe(const NActors::TActorContext& ctx, const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode);
- void WaitDataInPartition(const NActors::TActorContext& ctx);
- void SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx);
-
-private:
- const TActorId ParentId;
- const TString InternalClientId;
- const TString ClientDC;
- const ui64 Cookie;
- const TString Session;
- const ui32 Generation;
- const ui32 Step;
-
- NPersQueue::TTopicConverterPtr Topic;
- const ui32 Partition;
-
- const ui64 TabletID;
-
- ui64 ReadOffset;
- ui64 ClientReadOffset;
- ui64 ClientCommitOffset;
- bool ClientVerifyReadOffset;
- ui64 CommittedOffset;
- ui64 WriteTimestampEstimateMs;
-
- ui64 WTime;
- bool InitDone;
- bool StartReading;
- bool AllPrepareInited;
- bool FirstInit;
- TActorId PipeClient;
- ui32 PipeGeneration;
- bool RequestInfly;
- NKikimrClient::TPersQueueRequest CurrentRequest;
-
- ui64 EndOffset;
- ui64 SizeLag;
-
- TString ReadGuid; // empty if not reading
-
- bool NeedRelease;
- bool Released;
-
- std::set<ui64> WaitDataInfly;
- ui64 WaitDataCookie;
- bool WaitForData;
-
- bool LockCounted;
-
- std::deque<std::pair<ui64, ui64>> CommitsInfly; //ReadId, Offset
-
- TReadSessionActor::TTopicCounters Counters;
-};
-
-
-TReadSessionActor::TReadSessionActor(
- IReadSessionHandlerRef handler, const NPersQueue::TTopicsListController& topicsHandler, const ui64 cookie,
- const TActorId& pqMetaCache, const TActorId& newSchemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const TMaybe<TString> clientDC
-)
- : Handler(handler)
- , StartTimestamp(TInstant::Now())
- , PqMetaCache(pqMetaCache)
- , NewSchemeCache(newSchemeCache)
- , AuthInitActor()
- , AuthInflight(false)
- , ClientDC(clientDC ? *clientDC : "other")
- , ClientPath()
- , Session()
- , ClientsideLocksAllowed(false)
- , BalanceRightNow(false)
- , CommitsDisabled(false)
- , BalancersInitStarted(false)
- , InitDone(false)
- , ProtocolVersion(NPersQueue::TReadRequest::Base)
- , MaxReadMessagesCount(0)
- , MaxReadSize(0)
- , MaxReadPartitionsCount(0)
- , MaxTimeLagMs(0)
- , ReadTimestampMs(0)
- , ReadSettingsInited(false)
- , ForceACLCheck(false)
- , RequestNotChecked(true)
- , LastACLCheckTimestamp(TInstant::Zero())
- , ReadOnlyLocal(false)
- , ReadIdToResponse(1)
- , ReadIdCommitted(0)
- , LastCommitTimestamp(TInstant::Zero())
- , CommitInterval(DEFAULT_COMMIT_RATE)
- , CommitsInfly(0)
- , Cookie(cookie)
- , Counters(counters)
- , BytesInflight_(0)
- , RequestedBytes(0)
- , ReadsInfly(0)
- , TopicsHandler(topicsHandler)
-{
- Y_ASSERT(Handler);
-}
-
-
-
-TReadSessionActor::~TReadSessionActor() = default;
-
-
-void TReadSessionActor::Bootstrap(const TActorContext& ctx) {
- if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- ++(*GetServiceCounters(Counters, "pqproxy|readSession")->GetCounter("SessionsCreatedTotal", true));
- }
- StartTime = ctx.Now();
- Become(&TThis::StateFunc);
-}
-
-
-void TReadSessionActor::Die(const TActorContext& ctx) {
-
- ctx.Send(AuthInitActor, new TEvents::TEvPoisonPill());
-
- for (auto& p : Partitions) {
- ctx.Send(p.second.Actor, new TEvents::TEvPoisonPill());
-
- if (!p.second.Released) {
- auto it = TopicCounters.find(p.second.Converter->GetInternalName());
- Y_VERIFY(it != TopicCounters.end());
- it->second.PartitionsInfly.Dec();
- it->second.PartitionsReleased.Inc();
- if (p.second.Releasing)
- it->second.PartitionsToBeReleased.Dec();
- }
- }
-
- for (auto& t : Topics) {
- if (t.second.PipeClient)
- NTabletPipe::CloseClient(ctx, t.second.PipeClient);
- }
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " is DEAD");
-
- if (SessionsActive) {
- --(*SessionsActive);
- }
- if (BytesInflight) {
- (*BytesInflight) -= BytesInflight_;
- }
- if (SessionsActive) { //PartsPerSession is inited too
- PartsPerSession.DecFor(Partitions.size(), 1);
- }
- if (!Handler->IsShuttingDown())
- Handler->Finish();
- TActorBootstrapped<TReadSessionActor>::Die(ctx);
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& ctx) {
- CloseSession(TStringBuilder() << "Reads done signal - closing everything", NPersQueue::NErrorCode::OK, ctx);
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvWriteDone::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(BytesInflight_ >= ev->Get()->Size);
- BytesInflight_ -= ev->Get()->Size;
- if (BytesInflight) (*BytesInflight) -= ev->Get()->Size;
-
- const bool isAlive = ProcessReads(ctx);
- Y_UNUSED(isAlive);
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContext& ctx) {
- RequestNotChecked = true;
-
- if (CommitsDisabled) {
- CloseSession(TStringBuilder() << "commits in session are disabled by client option", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- const ui64 readId = ev->Get()->ReadId;
- if (readId <= ReadIdCommitted) {
- CloseSession(TStringBuilder() << "commit of " << ev->Get()->ReadId << " that is already committed", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- if (readId >= ReadIdToResponse) {
- CloseSession(TStringBuilder() << "commit of unknown cookie " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- if (NextCommits.size() >= AppData(ctx)->PQConfig.GetMaxReadCookies()) {
- CloseSession(TStringBuilder() << "got more than " << AppData(ctx)->PQConfig.GetMaxReadCookies() << " unordered cookies to commit " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- bool res = NextCommits.insert(readId).second;
- if (!res) {
- CloseSession(TStringBuilder() << "double commit of cookie " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from client for " << readId);
- MakeCommit(ctx);
-}
-
-void TReadSessionActor::MakeCommit(const TActorContext& ctx) {
- if (CommitsDisabled)
- return;
- if (ctx.Now() - LastCommitTimestamp < CommitInterval)
- return;
- if (CommitsInfly > MAX_COMMITS_INFLY)
- return;
- ui64 readId = ReadIdCommitted;
- auto it = NextCommits.begin();
- for (;it != NextCommits.end() && (*it) == readId + 1; ++it) {
- ++readId;
- }
- if (readId == ReadIdCommitted)
- return;
- NextCommits.erase(NextCommits.begin(), it);
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from " << ReadIdCommitted + 1 << " to " << readId);
-
- auto& commit = Commits[readId];
- commit.StartReadId = ReadIdCommitted + 1;
- commit.Partitions = 0;
- commit.StartTime = ctx.Now();
- ReadIdCommitted = readId;
- LastCommitTimestamp = ctx.Now();
- ++CommitsInfly;
- SLITotal.Inc();
- Y_VERIFY(Commits.size() == CommitsInfly);
-
- // Find last offset info belonging to our read id and its ancestors.
- const auto firstGreater = std::upper_bound(Offsets.begin(), Offsets.end(), readId);
- THashSet<std::pair<TString, ui64>> processedPartitions;
-
- // Iterate from last to first offsets to find partitions' offsets.
- // Offsets in queue have nondecreasing values (for each partition),
- // so it it sufficient to take only the last offset for each partition.
- // Note: reverse_iterator(firstGreater) points to _before_ firstGreater
-
- for (auto i = std::make_reverse_iterator(firstGreater), end = std::make_reverse_iterator(Offsets.begin()); i != end; ++i) {
- const TOffsetsInfo& info = *i;
- for (const TOffsetsInfo::TPartitionOffsetInfo& pi : info.PartitionOffsets) {
- if (!ActualPartitionActor(pi.Sender)) {
- continue;
- }
- const auto partitionKey = std::make_pair(pi.Topic, pi.Partition);
- if (!processedPartitions.insert(partitionKey).second) {
- continue; // already processed
- }
- const auto partitionIt = Partitions.find(partitionKey);
- if (partitionIt != Partitions.end() && !partitionIt->second.Released) {
- ctx.Send(partitionIt->second.Actor, new TEvPQProxy::TEvCommit(readId, pi.Offset));
- partitionIt->second.Commits.push_back(readId);
- ++commit.Partitions;
- }
- }
- }
- Offsets.erase(Offsets.begin(), firstGreater);
-
- AnswerForCommitsIfCan(ctx); //Could be done if all partitions are lost because of balancer dead
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvAuth::TPtr& ev, const TActorContext&) {
- ProcessAuth(ev->Get()->Auth);
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const TActorContext& ctx) {
-
- if (!ClientsideLocksAllowed) {
- CloseSession("Partition status available only when ClientsideLocksAllowed is true", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- auto it = Partitions.find(std::make_pair(ev->Get()->Topic, ev->Get()->Partition));
-
- if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) {
- //do nothing - already released partition
- LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL get status request from client for " << ev->Get()->Topic
- << ":" << ev->Get()->Partition << " generation " << ev->Get()->Generation);
- return;
- }
-
- //proxy request to partition - allow initing
- //TODO: add here VerifyReadOffset too and check it against Committed position
- ctx.Send(it->second.Actor, new TEvPQProxy::TEvGetStatus(ev->Get()->Topic, ev->Get()->Partition, ev->Get()->Generation));
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvLocked::TPtr& ev, const TActorContext& ctx) {
-
- RequestNotChecked = true;
- if (!ClientsideLocksAllowed) {
- CloseSession("Locked requests are allowed only when ClientsideLocksAllowed is true", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- auto& topic = ev->Get()->Topic;
- if (topic.empty()) {
- CloseSession("empty topic in start_read request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
-
- }
- auto it = Partitions.find(std::make_pair(topic, ev->Get()->Partition));
-
- if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) {
- //do nothing - already released partition
- LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL lock from client for " << topic
- << ":" << ev->Get()->Partition << " at offset " << ev->Get()->ReadOffset << " generation " << ev->Get()->Generation);
-
- return;
- }
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got lock from client for " << ev->Get()->Topic
- << ":" << ev->Get()->Partition << " at readOffset " << ev->Get()->ReadOffset << " commitOffset " << ev->Get()->CommitOffset << " generation " << ev->Get()->Generation);
-
- //proxy request to partition - allow initing
- //TODO: add here VerifyReadOffset too and check it against Committed position
- ctx.Send(it->second.Actor, new TEvPQProxy::TEvLockPartition(ev->Get()->ReadOffset, ev->Get()->CommitOffset, ev->Get()->VerifyReadOffset, true));
-}
-
-void TReadSessionActor::DropPartitionIfNeeded(THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator it, const TActorContext& ctx) {
- if (it->second.Commits.empty() && it->second.Released) {
- ctx.Send(it->second.Actor, new TEvents::TEvPoisonPill());
- bool res = ActualPartitionActors.erase(it->second.Actor);
- Y_VERIFY(res);
-
- if (--NumPartitionsFromTopic[it->second.Converter->GetInternalName()] == 0) {
- bool res = TopicCounters.erase(it->second.Converter->GetInternalName());
- Y_VERIFY(res);
- }
-
- if (SessionsActive) {
- PartsPerSession.DecFor(Partitions.size(), 1);
- }
- Partitions.erase(it);
- if (SessionsActive) {
- PartsPerSession.IncFor(Partitions.size(), 1);
- }
- }
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone::TPtr& ev, const TActorContext& ctx) {
-
- Y_VERIFY(!CommitsDisabled);
-
- if (!ActualPartitionActor(ev->Sender))
- return;
-
- ui64 readId = ev->Get()->ReadId;
-
- auto it = Commits.find(readId);
- Y_VERIFY(it != Commits.end());
- --it->second.Partitions;
-
- auto jt = Partitions.find(std::make_pair(ev->Get()->Topic->GetClientsideName(), ev->Get()->Partition));
- Y_VERIFY(jt != Partitions.end());
- Y_VERIFY(!jt->second.Commits.empty() && jt->second.Commits.front() == readId);
- jt->second.Commits.pop_front();
-
- DropPartitionIfNeeded(jt, ctx);
-
- AnswerForCommitsIfCan(ctx);
-
- MakeCommit(ctx);
-}
-
-void TReadSessionActor::AnswerForCommitsIfCan(const TActorContext& ctx) {
- while (!Commits.empty() && Commits.begin()->second.Partitions == 0) {
- auto it = Commits.begin();
- ui64 readId = it->first;
- TReadResponse result;
- for (ui64 i = it->second.StartReadId; i <= readId; ++i){
- result.MutableCommit()->AddCookie(i);
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits from " << it->second.StartReadId
- << " to " << readId);
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(result);
-
- ui32 commitDurationMs = (ctx.Now() - it->second.StartTime).MilliSeconds();
- CommitLatency.IncFor(commitDurationMs, 1);
- if (commitDurationMs >= AppData(ctx)->PQConfig.GetCommitLatencyBigMs()) {
- SLIBigLatency.Inc();
- }
- Commits.erase(it);
- --CommitsInfly;
- Y_VERIFY(Commits.size() == CommitsInfly);
- }
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvReadSessionStatus::TPtr& ev, const TActorContext& ctx) {
-
- THolder<TEvPQProxy::TEvReadSessionStatusResponse> result(new TEvPQProxy::TEvReadSessionStatusResponse());
- result->Record.SetSession(Session);
- result->Record.SetTimestamp(StartTimestamp.MilliSeconds());
-
- result->Record.SetClientNode(PeerName);
- result->Record.SetProxyNodeId(ctx.SelfID.NodeId());
-
- for (auto& p : Partitions) {
- auto part = result->Record.AddPartition();
- part->SetTopic(p.first.first);
- part->SetPartition(p.first.second);
- part->SetAssignId(0);
- for (auto& c : NextCommits) {
- part->AddNextCommits(c);
- }
- part->SetReadIdCommitted(ReadIdCommitted);
- part->SetLastReadId(ReadIdToResponse - 1);
- part->SetTimestampMs(0);
- }
-
- ctx.Send(ev->Sender, result.Release());
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorContext& ctx) {
-
- THolder<TEvPQProxy::TEvReadInit> event(ev->Release());
-
- if (!Topics.empty()) {
- //answer error
- CloseSession("got second init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- const auto& init = event->Request.GetInit();
-
- if (!init.TopicsSize()) {
- CloseSession("no topics in init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- if (init.GetClientId().empty()) {
- CloseSession("no clientId in init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- if (init.GetProxyCookie() != ctx.SelfID.NodeId() && init.GetProxyCookie() != MAGIC_COOKIE_VALUE) {
- CloseSession("you must perform ChooseProxy request at first and go to ProxyName server with ProxyCookie", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- // ToDo[migration] - consider separate consumer conversion logic - ?
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- ClientPath = init.GetClientId();
- ExternalClientId = ClientPath;
- InternalClientId = ConvertNewConsumerName(init.GetClientId());
- } else {
- ClientPath = NormalizeFullPath(MakeConsumerPath(init.GetClientId()));
- ExternalClientId = ClientPath;
- InternalClientId = ConvertNewConsumerName(init.GetClientId());
- }
-
- Auth = event->Request.GetCredentials();
- event->Request.ClearCredentials();
- Y_PROTOBUF_SUPPRESS_NODISCARD Auth.SerializeToString(&AuthStr);
- TStringBuilder session;
- session << ExternalClientId << "_" << ctx.SelfID.NodeId() << "_" << Cookie << "_" << TAppData::RandomProvider->GenRand64();
- Session = session;
- ProtocolVersion = init.GetProtocolVersion();
- CommitsDisabled = init.GetCommitsDisabled();
-
- if (ProtocolVersion >= NPersQueue::TReadRequest::ReadParamsInInit) {
- ReadSettingsInited = true;
- MaxReadMessagesCount = NormalizeMaxReadMessagesCount(init.GetMaxReadMessagesCount());
- MaxReadSize = NormalizeMaxReadSize(init.GetMaxReadSize());
- MaxReadPartitionsCount = NormalizeMaxReadPartitionsCount(init.GetMaxReadPartitionsCount());
- MaxTimeLagMs = init.GetMaxTimeLagMs();
- ReadTimestampMs = init.GetReadTimestampMs();
- }
-
- PeerName = event->PeerName;
- Database = event->Database;
-
- ReadOnlyLocal = init.GetReadOnlyLocal();
-
- if (init.GetCommitIntervalMs()) {
- CommitInterval = Min(CommitInterval, TDuration::MilliSeconds(init.GetCommitIntervalMs()));
- }
-
- for (ui32 i = 0; i < init.PartitionGroupsSize(); ++i) {
- Groups.push_back(init.GetPartitionGroups(i));
- }
- THashSet<TString> topicsToResolve;
- for (ui32 i = 0; i < init.TopicsSize(); ++i) {
- const auto& t = init.GetTopics(i);
-
- if (t.empty()) {
- CloseSession("empty topic in init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- topicsToResolve.insert(t);
- }
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName);
-
- ClientsideLocksAllowed = init.GetClientsideLocksAllowed();
- BalanceRightNow = init.GetBalancePartitionRightNow() || CommitsDisabled;
-
- if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- SetupCounters();
- }
-
- if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, "session without AuthInfo : " << ExternalClientId << " from " << PeerName);
- if (SessionsWithoutAuth) {
- ++(*SessionsWithoutAuth);
- }
- if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
- CloseSession("Unauthenticated access is forbidden, please provide credentials", NPersQueue::NErrorCode::ACCESS_DENIED, ctx);
- return;
- }
- }
- TopicsList = TopicsHandler.GetReadTopicsList(
- topicsToResolve, ReadOnlyLocal, Database
- );
- if (!TopicsList.IsValid) {
- return CloseSession(
- TopicsList.Reason,
- NPersQueue::NErrorCode::BAD_REQUEST, ctx
- );
- }
- SendAuthRequest(ctx);
-
- auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
- Aggr = {{{{"Account", ClientPath.substr(0, ClientPath.find("/"))}}, {"total"}}};
- SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
- SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
-
- SLITotal.Inc();
-}
-
-
-void TReadSessionActor::SendAuthRequest(const TActorContext& ctx) {
- AuthInitActor = {};
- AuthInflight = true;
-
- if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- Token = nullptr;
- CreateInitAndAuthActor(ctx);
- return;
- }
- auto database = Database.empty() ? NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig) : Database;
- Y_VERIFY(TopicsList.IsValid);
- TVector<TDiscoveryConverterPtr> topics;
- for(const auto& t : TopicsList.Topics) {
- if (topics.size() >= 10) {
- break;
- }
- topics.push_back(t.second);
- }
- ctx.Send(PqMetaCache, new TEvDescribeTopicsRequest(topics, false));
-}
-
-
-
-void TReadSessionActor::HandleDescribeTopicsResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
- TString dbId, folderId;
- for (const auto& entry : ev->Get()->Result->ResultSet) {
- Y_VERIFY(entry.PQGroupInfo); // checked at ProcessMetaCacheTopicResponse()
- auto& pqDescr = entry.PQGroupInfo->Description;
- dbId = pqDescr.GetPQTabletConfig().GetYdbDatabaseId();
- folderId = pqDescr.GetPQTabletConfig().GetYcFolderId();
- break;
- }
-
- auto entries = GetTicketParserEntries(dbId, folderId);
-
- TString ticket;
- switch (Auth.GetCredentialsCase()) {
- case NPersQueueCommon::TCredentials::kTvmServiceTicket:
- ticket = Auth.GetTvmServiceTicket();
- break;
- case NPersQueueCommon::TCredentials::kOauthToken:
- ticket = Auth.GetOauthToken();
- break;
- default:
- CloseSession("Unknown Credentials case", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- ctx.Send(MakeTicketParserID(), new TEvTicketParser::TEvAuthorizeTicket({
- .Database = Database,
- .Ticket = ticket,
- .PeerName = PeerName,
- .Entries = entries
- }));
-}
-
-void TReadSessionActor::CreateInitAndAuthActor(const TActorContext& ctx) {
- auto database = Database.empty() ? NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig) : Database;
- AuthInitActor = ctx.Register(new V1::TReadInitAndAuthActor(
- ctx, ctx.SelfID, InternalClientId, Cookie, Session, PqMetaCache, NewSchemeCache, Counters, Token,
- TopicsList, TopicsHandler.GetLocalCluster()
- ));
-}
-
-void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) {
- TString ticket = ev->Get()->Ticket;
- TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***";
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " << ev->Get()->Error << " user: "
- << (ev->Get()->Error.empty() ? ev->Get()->Token->GetUserSID() : ""));
-
- if (!ev->Get()->Error.empty()) {
- CloseSession(TStringBuilder() << "Ticket parsing error: " << ev->Get()->Error, NPersQueue::NErrorCode::ACCESS_DENIED, ctx);
- return;
- }
- Token = ev->Get()->Token;
- CreateInitAndAuthActor(ctx);
-}
-
-
-void TReadSessionActor::RegisterSession(const TActorId& pipe, const TString& topic, const TActorContext& ctx) {
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " register session to " << topic);
- THolder<TEvPersQueue::TEvRegisterReadSession> request;
- request.Reset(new TEvPersQueue::TEvRegisterReadSession);
- auto& req = request->Record;
- req.SetSession(Session);
- req.SetClientNode(PeerName);
- ActorIdToProto(pipe, req.MutablePipeClient());
- req.SetClientId(InternalClientId);
-
- for (ui32 i = 0; i < Groups.size(); ++i) {
- req.AddGroups(Groups[i]);
- }
-
- NTabletPipe::SendData(ctx, pipe, request.Release());
-}
-
-void TReadSessionActor::RegisterSessions(const TActorContext& ctx) {
- InitDone = true;
-
- for (auto& t : Topics) {
- auto& topic = t.first;
- RegisterSession(t.second.PipeClient, topic, ctx);
- NumPartitionsFromTopic[topic] = 0;
- }
-}
-
-
-void TReadSessionActor::SetupCounters()
-{
- if (SessionsCreated) {
- return;
- }
-
- auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession")->GetSubgroup("Client", InternalClientId)->GetSubgroup("ConsumerPath", ClientPath);
- SessionsCreated = subGroup->GetExpiringCounter("SessionsCreated", true);
- SessionsActive = subGroup->GetExpiringCounter("SessionsActive", false);
- SessionsWithoutAuth = subGroup->GetExpiringCounter("WithoutAuth", true);
- SessionsWithOldBatchingVersion = subGroup->GetExpiringCounter("SessionsWithOldBatchingVersion", true); // monitoring to ensure that old version is not used anymore
- Errors = subGroup->GetExpiringCounter("Errors", true);
- PipeReconnects = subGroup->GetExpiringCounter("PipeReconnects", true);
-
- BytesInflight = subGroup->GetExpiringCounter("BytesInflight", false);
-
- PartsPerSession = NKikimr::NPQ::TPercentileCounter(subGroup->GetSubgroup("sensor", "PartsPerSession"), {}, {}, "Count",
- TVector<std::pair<ui64, TString>>{{1, "1"}, {2, "2"}, {5, "5"},
- {10, "10"}, {20, "20"}, {50, "50"}, {70, "70"},
- {100, "100"}, {150, "150"}, {300,"300"}, {99999999, "99999999"}}, false);
-
- ++(*SessionsCreated);
- ++(*SessionsActive);
- PartsPerSession.IncFor(Partitions.size(), 1); //for 0
-
- if (ProtocolVersion < NPersQueue::TReadRequest::Batching) {
- ++(*SessionsWithOldBatchingVersion);
- }
-}
-
-
-void TReadSessionActor::SetupTopicCounters(const TTopicConverterPtr& topic)
-{
- auto& topicCounters = TopicCounters[topic->GetInternalName()];
- auto subGroup = GetServiceCounters(Counters, "pqproxy|readSession");
-//client/consumerPath Account/Producer OriginDC Topic/TopicPath
-
- auto aggr = GetLabels(topic);
- TVector<std::pair<TString, TString>> cons = {{"Client", InternalClientId}, {"ConsumerPath", ClientPath}};
-
- topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsLocked"}, true);
- topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsReleased"}, true);
- topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeReleased"}, false);
- topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsToBeLocked"}, false);
- topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsInfly"}, false);
- topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"PartitionsErrors"}, true);
- topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"Commits"}, true);
- topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"WaitsForData"}, true);
-}
-
-void TReadSessionActor::SetupTopicCounters(const TTopicConverterPtr& topic, const TString& cloudId,
- const TString& dbId, const TString& folderId)
-{
- auto& topicCounters = TopicCounters[topic->GetInternalName()];
- auto subGroup = NPersQueue::GetCountersForStream(Counters);
-//client/consumerPath Account/Producer OriginDC Topic/TopicPath
- auto aggr = GetLabelsForStream(topic, cloudId, dbId, folderId);
- TVector<std::pair<TString, TString>> cons = {{"consumer", ClientPath}};
-
- topicCounters.PartitionsLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked_per_second"}, true, "name");
- topicCounters.PartitionsReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_released_per_second"}, true, "name");
- topicCounters.PartitionsToBeReleased = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_released"}, false, "name");
- topicCounters.PartitionsToBeLocked = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_to_be_locked"}, false, "name");
- topicCounters.PartitionsInfly = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_locked"}, false, "name");
- topicCounters.Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.partitions_errors_per_second"}, true, "name");
- topicCounters.Commits = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.commits_per_second"}, true, "name");
- topicCounters.WaitsForData = NKikimr::NPQ::TMultiCounter(subGroup, aggr, cons, {"stream.internal_read.waits_for_data"}, true, "name");
-}
-
-void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) {
-
- LastACLCheckTimestamp = ctx.Now();
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone);
-
- AuthInitActor = TActorId();
- AuthInflight = false;
-
- if (!InitDone) {
-
- ui32 initBorder = AppData(ctx)->PQConfig.GetReadInitLatencyBigMs();
- ui32 readBorder = AppData(ctx)->PQConfig.GetReadLatencyBigMs();
- ui32 readBorderFromDisk = AppData(ctx)->PQConfig.GetReadLatencyFromDiskBigMs();
-
- auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
- InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadInit", initBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- CommitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Commit", AppData(ctx)->PQConfig.GetCommitLatencyBigMs(), {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false);
- ReadLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "Read", readBorder, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- ReadLatencyFromDisk = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "ReadFromDisk", readBorderFromDisk, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigReadLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadBigLatency"}, true, "sensor", false);
- ReadsTotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"ReadsTotal"}, true, "sensor", false);
-
- ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds();
- InitLatency.IncFor(initDurationMs, 1);
- if (initDurationMs >= initBorder) {
- SLIBigLatency.Inc();
- }
-
-
- TReadResponse result;
- result.MutableInit()->SetSessionId(Session);
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
-
- Handler->Reply(result);
-
- Handler->ReadyForNextRead();
-
- Y_VERIFY(!BalancersInitStarted);
- BalancersInitStarted = true;
-
- for (auto& t : ev->Get()->TopicAndTablets) {
- auto& topicHolder = Topics[t.TopicNameConverter->GetInternalName()];
- topicHolder.TabletID = t.TabletID;
- topicHolder.CloudId = t.CloudId;
- topicHolder.DbId = t.DbId;
- topicHolder.FolderId = t.FolderId;
- topicHolder.FullConverter = t.TopicNameConverter;
- FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter;
- const auto& second = t.TopicNameConverter->GetSecondaryPath();
- if (!second.empty()) {
- FullPathToConverter[second] = t.TopicNameConverter;
- }
- }
-
- for (auto& t : Topics) {
- NTabletPipe::TClientConfig clientConfig;
-
- clientConfig.CheckAliveness = false;
-
- clientConfig.RetryPolicy = RetryPolicyForPipes;
- t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig));
- }
-
- RegisterSessions(ctx);
-
- ctx.Schedule(Min(CommitInterval, CHECK_ACL_DELAY), new TEvents::TEvWakeup());
- } else {
- for (auto& t : ev->Get()->TopicAndTablets) {
- if (Topics.find(t.TopicNameConverter->GetInternalName()) == Topics.end()) {
- CloseSession(TStringBuilder() << "list of topics changed - new topic '" <<
- t.TopicNameConverter->GetPrimaryPath() << "' found",
- NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- }
- }
-}
-
-
-void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const TActorContext& ctx) {
-
- auto& record = ev->Get()->Record;
- Y_VERIFY(record.GetSession() == Session);
- Y_VERIFY(record.GetClientId() == InternalClientId);
-
- TActorId pipe = ActorIdFromProto(record.GetPipeClient());
- auto path = record.GetPath();
- if (path.empty()) {
- path = record.GetTopic();
- }
-
- auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path));
- if (converterIter.IsEnd()) {
- LOG_ALERT_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " ignored ev lock for event = " << record.ShortDebugString() << " path not recognized"
- );
- CloseSession(
- TStringBuilder() << "Internal server error, cannot parse lock event: " << record.ShortDebugString() << ", reason: topic not found",
- NPersQueue::NErrorCode::ERROR, ctx
- );
- return;
- }
- //auto topic = converterIter->second->GetClientsideName();
- auto intName = converterIter->second->GetInternalName();
- Y_VERIFY(!intName.empty());
- auto jt = Topics.find(intName);
-
- if (jt == Topics.end() || pipe != jt->second.PipeClient) { //this is message from old version of pipe
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " ignored ev lock for topic = " << converterIter->second->GetPrintableString()
- << " path recognized, but topic is unknown, this is unexpected"
- );
- return;
- }
- // ToDo[counters]
- if (NumPartitionsFromTopic[intName]++ == 0) {
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- SetupTopicCounters(converterIter->second, jt->second.CloudId, jt->second.DbId, jt->second.FolderId);
- } else {
- SetupTopicCounters(converterIter->second);
- }
- }
-
- auto it = TopicCounters.find(intName);
- Y_VERIFY(it != TopicCounters.end());
-
- IActor* partitionActor = new TPartitionActor(
- ctx.SelfID, InternalClientId, Cookie, Session, record.GetGeneration(),
- record.GetStep(), jt->second.FullConverter, record.GetPartition(), record.GetTabletId(), it->second,
- ClientDC
- );
-
- TActorId actorId = ctx.Register(partitionActor);
- if (SessionsActive) {
- PartsPerSession.DecFor(Partitions.size(), 1);
- }
- Y_VERIFY(record.GetGeneration() > 0);
- //Partitions use clientside name !
- auto pp = Partitions.insert({
- std::make_pair(jt->second.FullConverter->GetClientsideName(), record.GetPartition()),
- TPartitionActorInfo{actorId, (((ui64)record.GetGeneration()) << 32) + record.GetStep(), jt->second.FullConverter}
- });
- Y_VERIFY(pp.second);
- if (SessionsActive) {
- PartsPerSession.IncFor(Partitions.size(), 1);
- }
-
- bool res = ActualPartitionActors.insert(actorId).second;
- Y_VERIFY(res);
-
- it->second.PartitionsLocked.Inc();
- it->second.PartitionsInfly.Inc();
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " lock: " << record);
-
- ctx.Send(actorId, new TEvPQProxy::TEvLockPartition(0, 0, false, !ClientsideLocksAllowed));
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const TActorContext&) {
- if (!ActualPartitionActor(ev->Sender))
- return;
-
- auto& evTopic = ev->Get()->Topic;
- auto it = Partitions.find(std::make_pair(evTopic->GetClientsideName(), ev->Get()->Partition));
- Y_VERIFY(it != Partitions.end());
- Y_VERIFY(it->second.LockGeneration);
-
- if (it->second.Releasing) //lock request for already released partition - ignore
- return;
-
- if (ev->Get()->Init) {
- Y_VERIFY(!it->second.LockSent);
-
- it->second.LockSent = true;
- auto topicIter = Topics.find(evTopic->GetInternalName());
- Y_VERIFY(topicIter != Topics.end());
- Y_VERIFY(ClientsideLocksAllowed);
- TReadResponse result;
- auto lock = result.MutableLock();
- lock->SetTopic(topicIter->second.FullConverter->GetClientsideName());
- lock->SetPartition(ev->Get()->Partition);
- lock->SetReadOffset(ev->Get()->Offset);
- lock->SetEndOffset(ev->Get()->EndOffset);
- lock->SetGeneration(it->second.LockGeneration);
- auto jt = PartitionToReadResponse.find(it->second.Actor);
- if (jt == PartitionToReadResponse.end()) {
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(result);
- } else {
- jt->second->ControlMessages.push_back(result);
- }
- } else {
- Y_VERIFY(it->second.LockSent);
- TReadResponse result;
- auto status = result.MutablePartitionStatus();
- status->SetTopic(ev->Get()->Topic->GetClientsideName());
- status->SetPartition(ev->Get()->Partition);
- status->SetEndOffset(ev->Get()->EndOffset);
- status->SetGeneration(it->second.LockGeneration);
- status->SetCommittedOffset(ev->Get()->Offset);
- status->SetWriteWatermarkMs(ev->Get()->WriteTimestampEstimateMs);
- auto jt = PartitionToReadResponse.find(it->second.Actor);
- if (jt == PartitionToReadResponse.end()) {
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(result);
- } else {
- jt->second->ControlMessages.push_back(result);
- }
-
- }
-}
-
-void TReadSessionActor::Handle(TEvPersQueue::TEvError::TPtr& ev, const TActorContext& ctx) {
- CloseSession(ev->Get()->Record.GetDescription(), ev->Get()->Record.GetCode(), ctx);
-}
-
-
-void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, const TActorContext& ctx) {
- auto& record = ev->Get()->Record;
- Y_VERIFY(record.GetSession() == Session);
- Y_VERIFY(record.GetClientId() == InternalClientId);
- auto topic = record.GetPath();
- if (topic.empty()) {
- topic = record.GetTopic();
- }
- ui32 group = record.HasGroup() ? record.GetGroup() : 0;
-
- auto converterIter = FullPathToConverter.find(topic);
- if (converterIter.IsEnd()) {
- LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Failed to parse balancer response: " << record.ShortDebugString());
- CloseSession(
- TStringBuilder() << "Internal server error, cannot parse release event: " << record.ShortDebugString() << ", path not recognized",
- NPersQueue::NErrorCode::ERROR, ctx
- );
- return;
- }
- auto name = converterIter->second->GetInternalName();
-
- auto it = Topics.find(name);
- Y_VERIFY(it != Topics.end());
-
- TActorId pipe = ActorIdFromProto(record.GetPipeClient());
-
- if (pipe != it->second.PipeClient) { //this is message from old version of pipe
- return;
- }
-
- for (ui32 c = 0; c < record.GetCount(); ++c) {
- Y_VERIFY(!Partitions.empty());
-
- TActorId actorId = TActorId{};
- auto jt = Partitions.begin();
- ui32 i = 0;
- for (auto it = Partitions.begin(); it != Partitions.end(); ++it) {
- if (it->first.first == name && !it->second.Releasing && (group == 0 || it->first.second + 1 == group)) {
- ++i;
- if (rand() % i == 0) { //will lead to 1/n probability for each of n partitions
- actorId = it->second.Actor;
- jt = it;
- }
- }
- }
- Y_VERIFY(actorId);
-
- {
- auto it = TopicCounters.find(name);
- Y_VERIFY(it != TopicCounters.end());
- it->second.PartitionsToBeReleased.Inc();
- }
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " releasing " << jt->first.first << ":" << jt->first.second);
- jt->second.Releasing = true;
-
- ctx.Send(actorId, new TEvPQProxy::TEvReleasePartition());
- if (ClientsideLocksAllowed && jt->second.LockSent && !jt->second.Reading) { //locked and no active reads
- if (!ProcessReleasePartition(jt, BalanceRightNow, false, ctx)) { // returns false if actor died
- return;
- }
- }
- }
- AnswerForCommitsIfCan(ctx); // in case of killing partition
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionReleased::TPtr& ev, const TActorContext& ctx) {
- if (!ActualPartitionActor(ev->Sender))
- return;
-
- const auto& topic = ev->Get()->Topic;
- const ui32 partition = ev->Get()->Partition;
-
- auto jt = Partitions.find(std::make_pair(topic->GetClientsideName(), partition));
- Y_VERIFY(jt != Partitions.end(), "session %s topic %s part %u", Session.c_str(), topic->GetInternalName().c_str(), partition);
- Y_VERIFY(jt->second.Releasing);
- jt->second.Released = true;
-
- {
- auto it = TopicCounters.find(topic->GetInternalName());
- Y_VERIFY(it != TopicCounters.end());
- it->second.PartitionsReleased.Inc();
- it->second.PartitionsInfly.Dec();
- it->second.PartitionsToBeReleased.Dec();
-
- }
-
- InformBalancerAboutRelease(jt, ctx);
-
- DropPartitionIfNeeded(jt, ctx);
-}
-
-void TReadSessionActor::InformBalancerAboutRelease(const THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator& it, const TActorContext& ctx) {
-
- THolder<TEvPersQueue::TEvPartitionReleased> request;
- request.Reset(new TEvPersQueue::TEvPartitionReleased);
- auto& req = request->Record;
-
- auto jt = Topics.find(it->second.Converter->GetInternalName());
- Y_VERIFY(jt != Topics.end());
-
- req.SetSession(Session);
- ActorIdToProto(jt->second.PipeClient, req.MutablePipeClient());
- req.SetClientId(InternalClientId);
- req.SetTopic(it->first.first);
- req.SetPartition(it->first.second);
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->first.first << ":" << it->first.second);
-
- NTabletPipe::SendData(ctx, jt->second.PipeClient, request.Release());
-}
-
-
-void TReadSessionActor::CloseSession(const TString& errorReason, const NPersQueue::NErrorCode::EErrorCode errorCode, const NActors::TActorContext& ctx) {
-
- if (errorCode != NPersQueue::NErrorCode::OK) {
-
- if (InternalErrorCode(errorCode)) {
- SLIErrors.Inc();
- }
-
- if (Errors) {
- ++(*Errors);
- } else {
- if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- ++(*GetServiceCounters(Counters, "pqproxy|readSession")->GetCounter("Errors", true));
- }
- }
-
- TReadResponse result;
-
- auto error = result.MutableError();
- error->SetDescription(errorReason);
- error->SetCode(errorCode);
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed with error reason: " << errorReason);
- if (!Handler->IsShuttingDown()) {
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(result);
- } else {
- LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " GRps is shutting dows, skip reply");
- }
- } else {
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed");
- }
-
- Die(ctx);
-}
-
-
-void TReadSessionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
- TEvTabletPipe::TEvClientConnected *msg = ev->Get();
- if (msg->Status != NKikimrProto::OK) {
- if (msg->Dead) {
- CloseSession(TStringBuilder() << "one of topics is deleted, tablet " << msg->TabletId, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- //TODO: remove it
- CloseSession(TStringBuilder() << "unable to connect to one of topics, tablet " << msg->TabletId, NPersQueue::NErrorCode::ERROR, ctx);
- return;
-
- const bool isAlive = ProcessBalancerDead(msg->TabletId, ctx); // returns false if actor died
- Y_UNUSED(isAlive);
- return;
- }
-}
-
-bool TReadSessionActor::ActualPartitionActor(const TActorId& part) {
- return ActualPartitionActors.contains(part);
-}
-
-
-bool TReadSessionActor::ProcessReleasePartition(const THashMap<std::pair<TString, ui32>, TPartitionActorInfo>::iterator& it,
- bool kill, bool couldBeReads, const TActorContext& ctx)
-{
- //inform client
- if (ClientsideLocksAllowed && it->second.LockSent) {
- TReadResponse result;
- result.MutableRelease()->SetTopic(it->first.first);
- result.MutableRelease()->SetPartition(it->first.second);
- result.MutableRelease()->SetCanCommit(!kill);
- result.MutableRelease()->SetGeneration(it->second.LockGeneration);
- auto jt = PartitionToReadResponse.find(it->second.Actor);
- if (jt == PartitionToReadResponse.end()) {
- ui64 diff = result.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(result);
- } else {
- jt->second->ControlMessages.push_back(result);
- }
- it->second.LockGeneration = 0;
- it->second.LockSent = false;
- }
-
- if (!kill) {
- return true;
- }
-
- {
- auto jt = TopicCounters.find(it->second.Converter->GetInternalName());
- Y_VERIFY(jt != TopicCounters.end());
- jt->second.PartitionsReleased.Inc();
- jt->second.PartitionsInfly.Dec();
- if (!it->second.Released && it->second.Releasing) {
- jt->second.PartitionsToBeReleased.Dec();
- }
- }
-
- //process commits
- for (auto& c : it->second.Commits) {
- auto kt = Commits.find(c);
- Y_VERIFY(kt != Commits.end());
- --kt->second.Partitions;
- }
- it->second.Commits.clear();
-
- Y_VERIFY(couldBeReads || !it->second.Reading);
- //process reads
- TFormedReadResponse::TPtr formedResponseToAnswer;
- if (it->second.Reading) {
- const auto readIt = PartitionToReadResponse.find(it->second.Actor);
- Y_VERIFY(readIt != PartitionToReadResponse.end());
- if (--readIt->second->RequestsInfly == 0) {
- formedResponseToAnswer = readIt->second;
- }
- }
-
- InformBalancerAboutRelease(it, ctx);
-
- it->second.Released = true; //to force drop
- DropPartitionIfNeeded(it, ctx); //partition will be dropped
-
- if (formedResponseToAnswer) {
- return ProcessAnswer(ctx, formedResponseToAnswer); // returns false if actor died
- }
- return true;
-}
-
-
-bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorContext& ctx) {
- for (auto& t : Topics) {
- if (t.second.TabletID == tablet) {
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " balancer for topic " << t.first << " is dead, restarting all from this topic");
-
- //Drop all partitions from this topic
- for (auto it = Partitions.begin(); it != Partitions.end();) {
- if (it->second.Converter->GetInternalName() == t.first) { //partition from this topic
- // kill actor
- auto jt = it;
- ++it;
- if (!ProcessReleasePartition(jt, true, true, ctx)) { // returns false if actor died
- return false;
- }
- } else {
- ++it;
- }
- }
-
- AnswerForCommitsIfCan(ctx);
-
- //reconnect pipe
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.CheckAliveness = false;
- clientConfig.RetryPolicy = RetryPolicyForPipes;
- t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig));
- if (InitDone) {
- if (PipeReconnects) {
- ++(*PipeReconnects);
- ++(*Errors);
- }
-
- RegisterSession(t.second.PipeClient, t.first, ctx);
- }
- }
- }
- return true;
-}
-
-
-void TReadSessionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
- const bool isAlive = ProcessBalancerDead(ev->Get()->TabletId, ctx); // returns false if actor died
- Y_UNUSED(isAlive);
-}
-
-void TReadSessionActor::ProcessAuth(const NPersQueueCommon::TCredentials& auth) {
- TString tmp;
- Y_PROTOBUF_SUPPRESS_NODISCARD auth.SerializeToString(&tmp);
- if (auth.GetCredentialsCase() != NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET && tmp != AuthStr) {
- Auth = auth;
- AuthStr = tmp;
- ForceACLCheck = true;
- }
-}
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& ctx) {
- RequestNotChecked = true;
-
- THolder<TEvPQProxy::TEvRead> event(ev->Release());
-
- Handler->ReadyForNextRead();
-
-
- ProcessAuth(event->Request.GetCredentials());
- event->Request.ClearCredentials();
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got read request: " << event->Request.GetRead() << " with guid: " << event->Guid);
-
- Reads.emplace_back(event.Release());
-
- const bool isAlive = ProcessReads(ctx); // returns false if actor died
- Y_UNUSED(isAlive);
-}
-
-
-i64 TReadSessionActor::TFormedReadResponse::ApplyResponse(NPersQueue::TReadResponse&& resp) {
- Y_VERIFY(resp.GetBatchedData().PartitionDataSize() == 1);
- Response.MutableBatchedData()->AddPartitionData()->Swap(resp.MutableBatchedData()->MutablePartitionData(0));
- i64 prev = Response.ByteSize();
- std::swap<i64>(prev, ByteSize);
- return ByteSize - prev;
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TActorContext& ctx) {
- TActorId sender = ev->Sender;
- if (!ActualPartitionActor(sender))
- return;
-
- THolder<TEvPQProxy::TEvReadResponse> event(ev->Release());
-
- Y_VERIFY(event->Response.GetBatchedData().GetCookie() == 0); // cookie is not assigned
- Y_VERIFY(event->Response.GetBatchedData().PartitionDataSize() == 1);
-
- const TString topic = event->Response.GetBatchedData().GetPartitionData(0).GetTopic();
- const ui32 partition = event->Response.GetBatchedData().GetPartitionData(0).GetPartition();
- std::pair<TString, ui32> key(topic, partition);
- // Topic is expected to have clientSide name
- const auto partitionIt = Partitions.find(key);
- Y_VERIFY(partitionIt != Partitions.end());
- Y_VERIFY(partitionIt->second.Reading);
- partitionIt->second.Reading = false;
-
- auto it = PartitionToReadResponse.find(sender);
- Y_VERIFY(it != PartitionToReadResponse.end());
-
- TFormedReadResponse::TPtr formedResponse = it->second;
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " read done guid " << formedResponse->Guid
- << " " << key.first << ":" << key.second
- << " size " << event->Response.ByteSize());
-
- const i64 diff = formedResponse->ApplyResponse(std::move(event->Response));
- if (event->FromDisk) {
- formedResponse->FromDisk = true;
- }
- formedResponse->WaitQuotaTime = Max(formedResponse->WaitQuotaTime, event->WaitQuotaTime);
- --formedResponse->RequestsInfly;
- formedResponse->Offsets.PartitionOffsets.emplace_back(sender, topic, partition, event->NextReadOffset);
-
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
-
- if (ClientsideLocksAllowed && partitionIt->second.LockSent && partitionIt->second.Releasing) { //locked and need to be released
- if (!ProcessReleasePartition(partitionIt, BalanceRightNow, false, ctx)) { // returns false if actor died
- return;
- }
- }
- AnswerForCommitsIfCan(ctx); // in case of killing partition
-
- if (formedResponse->RequestsInfly == 0) {
- const bool isAlive = ProcessAnswer(ctx, formedResponse); // returns false if actor died
- Y_UNUSED(isAlive);
- }
-}
-
-
-bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadResponse::TPtr formedResponse) {
- ui32 readDurationMs = (ctx.Now() - formedResponse->Start - formedResponse->WaitQuotaTime).MilliSeconds();
- if (formedResponse->FromDisk) {
- ReadLatencyFromDisk.IncFor(readDurationMs, 1);
- } else {
- ReadLatency.IncFor(readDurationMs, 1);
- }
- if (readDurationMs >= (formedResponse->FromDisk ? AppData(ctx)->PQConfig.GetReadLatencyFromDiskBigMs() : AppData(ctx)->PQConfig.GetReadLatencyBigMs())) {
- SLIBigReadLatency.Inc();
- }
-
- Y_VERIFY(formedResponse->RequestsInfly == 0);
- i64 diff = formedResponse->Response.ByteSize();
- const bool hasMessages = RemoveEmptyMessages(*formedResponse->Response.MutableBatchedData());
- if (hasMessages) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " assign read id " << ReadIdToResponse << " to read request " << formedResponse->Guid);
- formedResponse->Response.MutableBatchedData()->SetCookie(ReadIdToResponse);
- // reply to client
- if (ProtocolVersion < NPersQueue::TReadRequest::Batching) {
- ConvertToOldBatch(formedResponse->Response);
- }
- diff -= formedResponse->Response.ByteSize(); // Bytes will be tracked inside handler
- Handler->Reply(formedResponse->Response);
- } else {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " empty read result " << formedResponse->Guid << ", start new reading");
- }
-
- BytesInflight_ -= diff;
- if (BytesInflight) (*BytesInflight) -= diff;
-
- for (auto& r : formedResponse->ControlMessages) {
- ui64 diff = r.ByteSize();
- BytesInflight_ += diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Handler->Reply(r);
- }
-
- for (const TActorId& p : formedResponse->PartitionsTookPartInRead) {
- PartitionToReadResponse.erase(p);
- }
-
- // Bring back available partitions.
- // If some partition was removed from partitions container, it is not bad because it will be checked during read processing.
- AvailablePartitions.insert(formedResponse->PartitionsBecameAvailable.begin(), formedResponse->PartitionsBecameAvailable.end());
-
- formedResponse->Offsets.ReadId = ReadIdToResponse;
-
- RequestedBytes -= formedResponse->RequestedBytes;
-
- ReadsInfly--;
-
- if (hasMessages) {
- if (!CommitsDisabled)
- Offsets.emplace_back(std::move(formedResponse->Offsets)); // even empty responses are needed for correct offsets commit.
- ReadIdToResponse++;
- } else {
- // process new read
- NPersQueue::TReadRequest req;
- req.MutableRead();
- Reads.emplace_back(new TEvPQProxy::TEvRead(req, formedResponse->Guid)); // Start new reading request with the same guid
- }
-
- return ProcessReads(ctx); // returns false if actor died
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvCloseSession::TPtr& ev, const TActorContext& ctx) {
- CloseSession(ev->Get()->Reason, ev->Get()->ErrorCode, ctx);
-}
-
-void TReadSessionActor::Handle(V1::TEvPQProxy::TEvCloseSession::TPtr& ev, const TActorContext& ctx) {
- CloseSession(ev->Get()->Reason, NErrorCode::EErrorCode(ev->Get()->ErrorCode - 500000), ctx);
-}
-
-ui32 TReadSessionActor::NormalizeMaxReadMessagesCount(ui32 sourceValue) {
- ui32 count = Min<ui32>(sourceValue, Max<i32>());
- if (count == 0) {
- count = Max<i32>();
- }
- return count;
-}
-
-ui32 TReadSessionActor::NormalizeMaxReadSize(ui32 sourceValue) {
- ui32 size = Min<ui32>(sourceValue, MAX_READ_SIZE);
- if (size == 0) {
- size = MAX_READ_SIZE;
- }
- return size;
-}
-
-ui32 TReadSessionActor::NormalizeMaxReadPartitionsCount(ui32 sourceValue) {
- ui32 maxPartitions = sourceValue;
- if (maxPartitions == 0) {
- maxPartitions = Max<ui32>();
- }
- return maxPartitions;
-}
-
-bool TReadSessionActor::CheckAndUpdateReadSettings(const NPersQueue::TReadRequest::TRead& readRequest) {
- if (ReadSettingsInited) { // already updated. Check that settings are not changed.
- const bool hasSettings = readRequest.GetMaxCount()
- || readRequest.GetMaxSize()
- || readRequest.GetPartitionsAtOnce()
- || readRequest.GetMaxTimeLagMs()
- || readRequest.GetReadTimestampMs();
- if (!hasSettings) {
- return true;
- }
-
- const bool settingsChanged = NormalizeMaxReadMessagesCount(readRequest.GetMaxCount()) != MaxReadMessagesCount
- || NormalizeMaxReadSize(readRequest.GetMaxSize()) != MaxReadSize
- || NormalizeMaxReadPartitionsCount(readRequest.GetPartitionsAtOnce()) != MaxReadPartitionsCount
- || readRequest.GetMaxTimeLagMs() != MaxTimeLagMs
- || readRequest.GetReadTimestampMs() != ReadTimestampMs;
- return !settingsChanged;
- } else {
- // Update settings for the first time
- ReadSettingsInited = true;
- MaxReadMessagesCount = NormalizeMaxReadMessagesCount(readRequest.GetMaxCount());
- MaxReadSize = NormalizeMaxReadSize(readRequest.GetMaxSize());
- MaxReadPartitionsCount = NormalizeMaxReadPartitionsCount(readRequest.GetPartitionsAtOnce());
- MaxTimeLagMs = readRequest.GetMaxTimeLagMs();
- ReadTimestampMs = readRequest.GetReadTimestampMs();
- return true;
- }
-}
-
-bool TReadSessionActor::ProcessReads(const TActorContext& ctx) {
- while (!Reads.empty() && BytesInflight_ + RequestedBytes < MAX_INFLY_BYTES && ReadsInfly < MAX_INFLY_READS) {
- const auto& readRequest = Reads.front()->Request.GetRead();
- if (!CheckAndUpdateReadSettings(readRequest)) {
- CloseSession("read settings were changed in read request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return false;
- }
-
- if (Offsets.size() >= AppData(ctx)->PQConfig.GetMaxReadCookies() + 10) {
- CloseSession(TStringBuilder() << "got more than " << AppData(ctx)->PQConfig.GetMaxReadCookies() << " uncommitted reads", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return false;
- }
-
- ui32 count = MaxReadMessagesCount;
- ui64 size = MaxReadSize;
- ui32 maxPartitions = MaxReadPartitionsCount;
- ui32 partitionsAsked = 0;
-
- TFormedReadResponse::TPtr formedResponse = new TFormedReadResponse(Reads.front()->Guid, ctx.Now());
- while (!AvailablePartitions.empty()) {
- auto part = *AvailablePartitions.begin();
- AvailablePartitions.erase(AvailablePartitions.begin());
-
- auto it = Partitions.find(std::make_pair(part.Topic->GetClientsideName(), part.Partition));
- if (it == Partitions.end() || it->second.Releasing || it->second.Actor != part.Actor) { //this is already released partition
- continue;
- }
- //add this partition to reading
- ++partitionsAsked;
-
- TAutoPtr<TEvPQProxy::TEvRead> read = new TEvPQProxy::TEvRead(Reads.front()->Request, Reads.front()->Guid);
- const ui32 ccount = Min<ui32>(part.MsgLag * LAG_GROW_MULTIPLIER, count);
- count -= ccount;
- const ui64 csize = (ui64)Min<double>(part.SizeLag * LAG_GROW_MULTIPLIER, size);
- size -= csize;
-
- Y_VERIFY(csize < Max<i32>());
- auto* readR = read->Request.MutableRead();
- readR->SetMaxCount(ccount);
- readR->SetMaxSize(csize);
- readR->SetMaxTimeLagMs(MaxTimeLagMs);
- readR->SetReadTimestampMs(ReadTimestampMs);
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX
- << " performing read request: " << (*readR) << " with guid " << read->Guid
- << " from " << part.Topic->GetPrintableString() << ", partition:" << part.Partition
- << " count " << ccount << " size " << csize
- << " partitionsAsked " << partitionsAsked << " maxTimeLag " << MaxTimeLagMs << "ms");
-
-
- Y_VERIFY(!it->second.Reading);
- it->second.Reading = true;
- formedResponse->PartitionsTookPartInRead.insert(it->second.Actor);
-
- RequestedBytes += csize;
- formedResponse->RequestedBytes += csize;
-
- ctx.Send(it->second.Actor, read.Release());
- const auto insertResult = PartitionToReadResponse.insert(std::make_pair(it->second.Actor, formedResponse));
- Y_VERIFY(insertResult.second);
-
- if (--maxPartitions == 0 || count == 0 || size == 0)
- break;
- }
- if (partitionsAsked == 0)
- break;
- ReadsTotal.Inc();
- formedResponse->RequestsInfly = partitionsAsked;
-
- ReadsInfly++;
-
- i64 diff = formedResponse->Response.ByteSize();
- BytesInflight_ += diff;
- formedResponse->ByteSize = diff;
- if (BytesInflight) (*BytesInflight) += diff;
- Reads.pop_front();
- }
- return true;
-}
-
-
-void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionReady::TPtr& ev, const TActorContext& ctx) {
-
- if (!ActualPartitionActor(ev->Sender))
- return;
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << ev->Get()->Topic->GetPrintableString()
- << " partition:" << ev->Get()->Partition << " ready for read with readOffset "
- << ev->Get()->ReadOffset << " endOffset " << ev->Get()->EndOffset << " WTime "
- << ev->Get()->WTime << " sizeLag " << ev->Get()->SizeLag);
-
- const auto it = PartitionToReadResponse.find(ev->Sender); // check whether this partition is taking part in read response
- auto& container = it != PartitionToReadResponse.end() ? it->second->PartitionsBecameAvailable : AvailablePartitions;
- auto res = container.insert({ev->Get()->Topic, ev->Get()->Partition, ev->Get()->WTime, ev->Get()->SizeLag,
- ev->Get()->EndOffset - ev->Get()->ReadOffset, ev->Sender});
- Y_VERIFY(res.second);
- const bool isAlive = ProcessReads(ctx); // returns false if actor died
- Y_UNUSED(isAlive);
-}
-
-
-void TReadSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const TActorContext& ctx) {
- CloseSession(ev->Get()->Reason, ev->Get()->ErrorCode, ctx);
-}
-
-
-void TReadSessionActor::HandleWakeup(const TActorContext& ctx) {
- ctx.Schedule(Min(CommitInterval, CHECK_ACL_DELAY), new TEvents::TEvWakeup());
- MakeCommit(ctx);
- if (!AuthInflight && (ForceACLCheck || (ctx.Now() - LastACLCheckTimestamp > TDuration::Seconds(AppData(ctx)->PQConfig.GetACLRetryTimeoutSec()) && RequestNotChecked))) {
- ForceACLCheck = false;
- RequestNotChecked = false;
- Y_VERIFY(!AuthInitActor);
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout");
-
- SendAuthRequest(ctx);
- }
-}
-
-bool TReadSessionActor::RemoveEmptyMessages(TReadResponse::TBatchedData& data) {
- bool hasNonEmptyMessages = false;
- auto isMessageEmpty = [&](TReadResponse::TBatchedData::TMessageData& message) -> bool {
- if (message.GetData().empty()) {
- return true;
- } else {
- hasNonEmptyMessages = true;
- return false;
- }
- };
- auto batchRemover = [&](TReadResponse::TBatchedData::TBatch& batch) -> bool {
- NProtoBuf::RemoveRepeatedFieldItemIf(batch.MutableMessageData(), isMessageEmpty);
- return batch.MessageDataSize() == 0;
- };
- auto partitionDataRemover = [&](TReadResponse::TBatchedData::TPartitionData& partition) -> bool {
- NProtoBuf::RemoveRepeatedFieldItemIf(partition.MutableBatch(), batchRemover);
- return partition.BatchSize() == 0;
- };
- NProtoBuf::RemoveRepeatedFieldItemIf(data.MutablePartitionData(), partitionDataRemover);
- return hasNonEmptyMessages;
-}
-
-
-////////////////// PARTITION ACTOR
-
-TPartitionActor::TPartitionActor(
- const TActorId& parentId, const TString& internalClientId, const ui64 cookie, const TString& session,
- const ui32 generation, const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const ui32 partition,
- const ui64 tabletID, const TReadSessionActor::TTopicCounters& counters, const TString& clientDC
-)
- : ParentId(parentId)
- , InternalClientId(internalClientId)
- , ClientDC(clientDC)
- , Cookie(cookie)
- , Session(session)
- , Generation(generation)
- , Step(step)
- , Topic(topic)
- , Partition(partition)
- , TabletID(tabletID)
- , ReadOffset(0)
- , ClientReadOffset(0)
- , ClientCommitOffset(0)
- , ClientVerifyReadOffset(false)
- , CommittedOffset(0)
- , WriteTimestampEstimateMs(0)
- , WTime(0)
- , InitDone(false)
- , StartReading(false)
- , AllPrepareInited(false)
- , FirstInit(true)
- , PipeClient()
- , PipeGeneration(0)
- , RequestInfly(false)
- , EndOffset(0)
- , SizeLag(0)
- , NeedRelease(false)
- , Released(false)
- , WaitDataCookie(0)
- , WaitForData(false)
- , LockCounted(false)
- , Counters(counters)
-{
-}
-
-
-TPartitionActor::~TPartitionActor() = default;
-
-
-void TPartitionActor::Bootstrap(const TActorContext&) {
- Become(&TThis::StateFunc);
-}
-
-
-void TPartitionActor::CheckRelease(const TActorContext& ctx) {
- const bool hasUncommittedData = ReadOffset > ClientCommitOffset && ReadOffset > ClientReadOffset;
- if (NeedRelease) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " checking release readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid
- << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released);
- }
-
- if (NeedRelease && ReadGuid.empty() && CommitsInfly.empty() && !hasUncommittedData && !Released) {
- Released = true;
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReleased(Topic, Partition));
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " check release done - releasing; readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid
- << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released);
-
- }
-}
-
-
-void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) {
- NKikimrClient::TPersQueueRequest request;
- request.MutablePartitionRequest()->SetTopic(Topic->GetClientsideName());
- request.MutablePartitionRequest()->SetPartition(Partition);
- request.MutablePartitionRequest()->SetCookie(readId);
-
- Y_VERIFY(PipeClient);
-
- ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
- auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset();
- commit->SetClientId(InternalClientId);
- commit->SetOffset(offset);
- Y_VERIFY(!Session.empty());
- commit->SetSessionId(Session);
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:"
- << Partition << " committing to position " << offset << " prev " << CommittedOffset
- << " end " << EndOffset << " by cookie " << readId);
-
- TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest);
- req->Record.Swap(&request);
-
- NTabletPipe::SendData(ctx, PipeClient, req.Release());
-}
-
-void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) {
-
- if (!PipeClient)
- return;
-
- Counters.Errors.Inc();
-
- NTabletPipe::CloseClient(ctx, PipeClient);
- PipeClient = TActorId{};
- if (errorCode != NPersQueue::NErrorCode::OVERLOAD)
- ++PipeGeneration;
-
- if (PipeGeneration == MAX_PIPE_RESTARTS) {
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("too much attempts to restart pipe", NPersQueue::NErrorCode::ERROR));
- return;
- }
-
- ctx.Schedule(TDuration::MilliSeconds(RESTART_PIPE_DELAY_MS), new TEvPQProxy::TEvRestartPipe());
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " schedule pipe restart attempt " << PipeGeneration << " reason: " << reason);
-}
-
-
-void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TActorContext& ctx) {
-
- Y_VERIFY(!PipeClient);
-
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
- PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig));
- Y_VERIFY(TabletID);
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " pipe restart attempt " << PipeGeneration << " RequestInfly " << RequestInfly << " ReadOffset " << ReadOffset << " EndOffset " << EndOffset
- << " InitDone " << InitDone << " WaitForData " << WaitForData);
-
- if (RequestInfly) { //got read infly
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " resend " << CurrentRequest);
-
- TAutoPtr<TEvPersQueue::TEvRequest> event(new TEvPersQueue::TEvRequest);
- event->Record = CurrentRequest;
-
- ActorIdToProto(PipeClient, event->Record.MutablePartitionRequest()->MutablePipeClient());
-
- NTabletPipe::SendData(ctx, PipeClient, event.Release());
- }
- if (InitDone) {
- for (auto& c : CommitsInfly) { //resend all commits
- if (c.second != Max<ui64>())
- SendCommit(c.first, c.second, ctx);
- }
- if (WaitForData) { //resend wait-for-data requests
- WaitDataInfly.clear();
- WaitDataInPartition(ctx);
- }
- }
-}
-
-void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) {
-
- if (ev->Get()->Record.HasErrorCode() && ev->Get()->Record.GetErrorCode() != NPersQueue::NErrorCode::OK) {
- const auto errorCode = ev->Get()->Record.GetErrorCode();
- if (errorCode == NPersQueue::NErrorCode::WRONG_COOKIE || errorCode == NPersQueue::NErrorCode::BAD_REQUEST) {
- Counters.Errors.Inc();
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + ev->Get()->Record.GetErrorReason(), ev->Get()->Record.GetErrorCode()));
- } else {
- RestartPipe(ctx, TStringBuilder() << "status is not ok. Code: " << EErrorCode_Name(errorCode) << ". Reason: " << ev->Get()->Record.GetErrorReason(), errorCode);
- }
- return;
- }
-
- if (ev->Get()->Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { //this is incorrect answer, die
- Y_VERIFY(!ev->Get()->Record.HasErrorCode());
- Counters.Errors.Inc();
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + ev->Get()->Record.GetErrorReason(), NPersQueue::NErrorCode::ERROR));
- return;
- }
- if (!ev->Get()->Record.HasPartitionResponse()) { //this is incorrect answer, die
- Counters.Errors.Inc();
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("empty partition in response", NPersQueue::NErrorCode::ERROR));
- return;
- }
-
- const auto& result = ev->Get()->Record.GetPartitionResponse();
-
- if (!result.HasCookie()) { //this is incorrect answer, die
- Counters.Errors.Inc();
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("no cookie in response", NPersQueue::NErrorCode::ERROR));
- return;
- }
-
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " initDone " << InitDone << " event " << PartitionResponseToLog(result));
-
-
- if (!InitDone) {
- if (result.GetCookie() != INIT_COOKIE) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " unwaited response in init with cookie " << result.GetCookie());
- return;
- }
- Y_VERIFY(RequestInfly);
- CurrentRequest.Clear();
- RequestInfly = false;
-
- Y_VERIFY(result.HasCmdGetClientOffsetResult());
- const auto& resp = result.GetCmdGetClientOffsetResult();
- Y_VERIFY(resp.HasEndOffset());
- EndOffset = resp.GetEndOffset();
- SizeLag = resp.GetSizeLag();
-
- ClientCommitOffset = ReadOffset = CommittedOffset = resp.HasOffset() ? resp.GetOffset() : 0;
- Y_VERIFY(EndOffset >= CommittedOffset);
-
- if (resp.HasWriteTimestampMS())
- WTime = resp.GetWriteTimestampMS();
- WriteTimestampEstimateMs = resp.GetWriteTimestampEstimateMS();
- InitDone = true;
- PipeGeneration = 0; //reset tries counter - all ok
- LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " INIT DONE " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " << CommittedOffset);
-
-
- if (!StartReading) {
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Topic, Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, true));
- } else {
- InitStartReading(ctx);
- }
- return;
- }
-
- if (!result.HasCmdReadResult()) { //this is commit response
- if (CommitsInfly.empty()) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for nothing");
- return;
- }
- ui64 readId = CommitsInfly.front().first;
-
- if (result.GetCookie() != readId) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for " << readId);
- return;
- }
-
- Counters.Commits.Inc();
-
- CommittedOffset = CommitsInfly.front().second;
- CommitsInfly.pop_front();
- if (readId != Max<ui64>()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset
- ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition));
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId);
-
- while (!CommitsInfly.empty() && CommitsInfly.front().second == Max<ui64>()) { //this is cookies that have no effect on this partition
- readId = CommitsInfly.front().first;
- CommitsInfly.pop_front();
- ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition));
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << "partition :" << Partition
- << " commit done with no effect with cookie " << readId);
- }
-
- CheckRelease(ctx);
- PipeGeneration = 0; //reset tries counter - all ok
- return;
- }
-
- //This is read
-
- Y_VERIFY(result.HasCmdReadResult());
- const auto& res = result.GetCmdReadResult();
-
- if (result.GetCookie() != (ui64)ReadOffset) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString()
- << "partition :" << Partition
- << " unwaited read-response with cookie " << result.GetCookie() << "; waiting for " << ReadOffset << "; current read guid is " << ReadGuid);
- return;
- }
-
- Y_VERIFY(res.HasMaxOffset());
- EndOffset = res.GetMaxOffset();
- SizeLag = res.GetSizeLag();
-
- const ui64 realReadOffset = res.HasRealReadOffset() ? res.GetRealReadOffset() : 0;
-
- TReadResponse response;
-
- auto* data = response.MutableBatchedData();
- auto* partitionData = data->AddPartitionData();
- partitionData->SetTopic(Topic->GetClientsideName());
- partitionData->SetPartition(Partition);
-
- bool hasOffset = false;
-
- TReadResponse::TBatchedData::TBatch* currentBatch = nullptr;
- for (ui32 i = 0; i < res.ResultSize(); ++i) {
- const auto& r = res.GetResult(i);
-
- WTime = r.GetWriteTimestampMS();
- WriteTimestampEstimateMs = Max(WriteTimestampEstimateMs, WTime);
- Y_VERIFY(r.GetOffset() >= ReadOffset);
- ReadOffset = r.GetOffset() + 1;
- hasOffset = true;
-
- auto proto(GetDeserializedData(r.GetData()));
- if (proto.GetChunkType() != NKikimrPQClient::TDataChunk::REGULAR) {
- continue; //TODO - no such chunks must be on prod
- }
-
- Y_VERIFY(!r.GetSourceId().empty());
- if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) {
- LOG_ERROR_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << "read bad sourceId from topic " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "' ReadGuid " << ReadGuid);
- }
- TString sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId());
-
- if (!currentBatch || currentBatch->GetWriteTimeMs() != r.GetWriteTimestampMS() || currentBatch->GetSourceId() != sourceId) {
- // If write time and source id are the same, the rest fields will be the same too.
- currentBatch = partitionData->AddBatch();
- currentBatch->SetWriteTimeMs(r.GetWriteTimestampMS());
- currentBatch->SetSourceId(sourceId);
-
- if (proto.HasMeta()) {
- const auto& header = proto.GetMeta();
- if (header.HasServer()) {
- auto* item = currentBatch->MutableExtraFields()->AddItems();
- item->SetKey("server");
- item->SetValue(header.GetServer());
- }
- if (header.HasFile()) {
- auto* item = currentBatch->MutableExtraFields()->AddItems();
- item->SetKey("file");
- item->SetValue(header.GetFile());
- }
- if (header.HasIdent()) {
- auto* item = currentBatch->MutableExtraFields()->AddItems();
- item->SetKey("ident");
- item->SetValue(header.GetIdent());
- }
- if (header.HasLogType()) {
- auto* item = currentBatch->MutableExtraFields()->AddItems();
- item->SetKey("logtype");
- item->SetValue(header.GetLogType());
- }
- }
-
- if (proto.HasExtraFields()) {
- const auto& map = proto.GetExtraFields();
- for (const auto& kv : map.GetItems()) {
- auto* item = currentBatch->MutableExtraFields()->AddItems();
- item->SetKey(kv.GetKey());
- item->SetValue(kv.GetValue());
- }
- }
-
- if (proto.HasIp() && IsUtf(proto.GetIp())) {
- currentBatch->SetIp(proto.GetIp());
- }
- }
-
- auto* message = currentBatch->AddMessageData();
- message->SetSeqNo(r.GetSeqNo());
- message->SetCreateTimeMs(r.GetCreateTimestampMS());
- message->SetOffset(r.GetOffset());
- message->SetUncompressedSize(r.GetUncompressedSize());
- if (proto.HasCodec()) {
- const auto codec = proto.GetCodec();
- if (codec < Min<int>() || codec > Max<int>() || !NPersQueueCommon::ECodec_IsValid(codec)) {
- LOG_ERROR_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << "data chunk (topic " << Topic->GetInternalName() << ", partition " << Partition
- << ", offset " << r.GetOffset() << ", seqNo " << r.GetSeqNo() << ", sourceId "
- << r.GetSourceId() << ") codec (id " << codec
- << ") is not valid NPersQueueCommon::ECodec, loss of data compression codec information"
- );
- }
- message->SetCodec((NPersQueueCommon::ECodec)proto.GetCodec());
- }
- message->SetData(proto.GetData());
- }
-
- if (!hasOffset) { //no data could be read from paritition at offset ReadOffset - no data in partition at all???
- ReadOffset = Min(Max(ReadOffset + 1, realReadOffset + 1), EndOffset);
- }
-
- CurrentRequest.Clear();
- RequestInfly = false;
-
- Y_VERIFY(!WaitForData);
-
- if (EndOffset > ReadOffset) {
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReady(Topic, Partition, WTime, SizeLag, ReadOffset, EndOffset));
- } else {
- WaitForData = true;
- if (PipeClient) //pipe will be recreated soon
- WaitDataInPartition(ctx);
- }
-
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " after read state " << Topic->GetPrintableString()
- << " partition:" << Partition
- << " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid);
-
- ReadGuid = TString();
-
- auto readResponse = MakeHolder<TEvPQProxy::TEvReadResponse>(
- std::move(response),
- ReadOffset,
- res.GetBlobsFromDisk() > 0,
- TDuration::MilliSeconds(res.GetWaitQuotaTimeMs())
- );
- ctx.Send(ParentId, readResponse.Release());
- CheckRelease(ctx);
-
- PipeGeneration = 0; //reset tries counter - all ok
-}
-
-void TPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
- TEvTabletPipe::TEvClientConnected *msg = ev->Get();
-
- LOG_INFO_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " pipe restart attempt " << PipeGeneration << " pipe creation result: " << msg->Status);
-
- if (msg->Status != NKikimrProto::OK) {
- RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << msg->TabletId, NPersQueue::NErrorCode::ERROR);
- return;
- }
-}
-
-void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
- RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << ev->Get()->TabletId, NPersQueue::NErrorCode::ERROR);
-}
-
-
-void TPartitionActor::Handle(TEvPQProxy::TEvReleasePartition::TPtr&, const TActorContext& ctx) {
- LOG_INFO_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " (partition)releasing " << Topic->GetPrintableString() << " partition:" << Partition
- << " ReadOffset " << ReadOffset << " ClientCommitOffset " << ClientCommitOffset
- << " CommittedOffst " << CommittedOffset
- );
- NeedRelease = true;
- CheckRelease(ctx);
-}
-
-
-void TPartitionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr&, const TActorContext& ctx) {
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Topic, Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, false));
-}
-
-
-void TPartitionActor::Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const TActorContext& ctx) {
- ClientReadOffset = ev->Get()->ReadOffset;
- ClientCommitOffset = ev->Get()->CommitOffset;
- ClientVerifyReadOffset = ev->Get()->VerifyReadOffset;
-
- if (StartReading) {
- Y_VERIFY(ev->Get()->StartReading); //otherwise it is signal from actor, this could not be done
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("double partition locking", NPersQueue::NErrorCode::BAD_REQUEST));
- return;
- }
-
- StartReading = ev->Get()->StartReading;
- InitLockPartition(ctx);
-}
-
-void TPartitionActor::InitStartReading(const TActorContext& ctx) {
-
- Y_VERIFY(AllPrepareInited);
- Y_VERIFY(!WaitForData);
- LOG_INFO_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " Start reading " << Topic->GetPrintableString() << " partition:" << Partition
- << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset "
- << CommittedOffset << " clientCommittedOffset " << ClientCommitOffset
- << " clientReadOffset " << ClientReadOffset
- );
-
- Counters.PartitionsToBeLocked.Dec();
- LockCounted = false;
-
- ReadOffset = Max<ui64>(CommittedOffset, ClientReadOffset);
-
- if (ClientVerifyReadOffset) {
- if (ClientReadOffset < CommittedOffset) {
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession(TStringBuilder()
- << "trying to read from position that is less than committed: read " << ClientReadOffset << " committed " << CommittedOffset,
- NPersQueue::NErrorCode::BAD_REQUEST));
- return;
- }
- }
-
- if (ClientCommitOffset > CommittedOffset) {
- if (ClientCommitOffset > ReadOffset) {
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession(TStringBuilder()
- << "trying to read from position that is less than provided to commit: read " << ReadOffset << " commit " << ClientCommitOffset,
- NPersQueue::NErrorCode::BAD_REQUEST));
- return;
- }
- if (ClientCommitOffset > EndOffset) {
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession(TStringBuilder()
- << "trying to commit to future: commit " << ClientCommitOffset << " endOffset " << EndOffset,
- NPersQueue::NErrorCode::BAD_REQUEST));
- return;
- }
- Y_VERIFY(CommitsInfly.empty());
- CommitsInfly.push_back(std::pair<ui64, ui64>(Max<ui64>(), ClientCommitOffset));
- if (PipeClient) //pipe will be recreated soon
- SendCommit(CommitsInfly.back().first, CommitsInfly.back().second, ctx);
- } else {
- ClientCommitOffset = CommittedOffset;
- }
-
- if (EndOffset > ReadOffset) {
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReady(Topic, Partition, WTime, SizeLag, ReadOffset, EndOffset));
- } else {
- WaitForData = true;
- if (PipeClient) //pipe will be recreated soon
- WaitDataInPartition(ctx);
- }
-}
-
-void TPartitionActor::InitLockPartition(const TActorContext& ctx) {
- if (PipeClient && AllPrepareInited) {
- ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("double partition locking", NPersQueue::NErrorCode::BAD_REQUEST));
- return;
- }
- if (!LockCounted) {
- Counters.PartitionsToBeLocked.Inc();
- LockCounted = true;
- }
- if (StartReading)
- AllPrepareInited = true;
-
- if (FirstInit) {
- Y_VERIFY(!PipeClient);
- FirstInit = false;
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
- PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig));
-
- NKikimrClient::TPersQueueRequest request;
-
- request.MutablePartitionRequest()->SetTopic(Topic->GetClientsideName());
- request.MutablePartitionRequest()->SetPartition(Partition);
- request.MutablePartitionRequest()->SetCookie(INIT_COOKIE);
-
- ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
-
- auto cmd = request.MutablePartitionRequest()->MutableCmdCreateSession();
- cmd->SetClientId(InternalClientId);
- cmd->SetSessionId(Session);
- cmd->SetGeneration(Generation);
- cmd->SetStep(Step);
-
- LOG_INFO_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " INITING " << Topic->GetPrintableString() << " partition:" << Partition);
-
- TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest);
- Y_VERIFY(!RequestInfly);
- CurrentRequest = request;
- RequestInfly = true;
- req->Record.Swap(&request);
-
- NTabletPipe::SendData(ctx, PipeClient, req.Release());
- } else {
- Y_VERIFY(StartReading); //otherwise it is double locking from actor, not client - client makes lock always with StartReading == true
- Y_VERIFY(InitDone);
- InitStartReading(ctx);
- }
-}
-
-
-void TPartitionActor::WaitDataInPartition(const TActorContext& ctx) {
-
- if (WaitDataInfly.size() > 1) //already got 2 requests inflight
- return;
- Y_VERIFY(InitDone);
-
- Y_VERIFY(PipeClient);
-
- if (!WaitForData)
- return;
-
- Y_VERIFY(ReadOffset >= EndOffset);
-
- TAutoPtr<TEvPersQueue::TEvHasDataInfo> event(new TEvPersQueue::TEvHasDataInfo());
- event->Record.SetPartition(Partition);
- event->Record.SetOffset(ReadOffset);
- event->Record.SetCookie(++WaitDataCookie);
- ui64 deadline = (ctx.Now() + WAIT_DATA - WAIT_DELTA).MilliSeconds();
- event->Record.SetDeadline(deadline);
- event->Record.SetClientId(InternalClientId);
-
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " wait data in partition inited, cookie " << WaitDataCookie
- );
-
- NTabletPipe::SendData(ctx, PipeClient, event.Release());
-
- ctx.Schedule(PREWAIT_DATA, new TEvents::TEvWakeup());
-
- ctx.Schedule(WAIT_DATA, new TEvPQProxy::TEvDeadlineExceeded(WaitDataCookie));
-
- WaitDataInfly.insert(WaitDataCookie);
-}
-
-void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, const TActorContext& ctx) {
- const auto& record = ev->Get()->Record;
-
- WriteTimestampEstimateMs = Max(WriteTimestampEstimateMs, record.GetWriteTimestampEstimateMS());
-
- auto it = WaitDataInfly.find(ev->Get()->Record.GetCookie());
- if (it == WaitDataInfly.end()) {
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " unwaited response for WaitData " << ev->Get()->Record);
- return;
- }
- WaitDataInfly.erase(it);
- if (!WaitForData)
- return;
-
- Counters.WaitsForData.Inc();
-
- Y_VERIFY(record.HasEndOffset());
- Y_VERIFY(EndOffset <= record.GetEndOffset()); //end offset could not be changed if no data arrived, but signal will be sended anyway after timeout
- Y_VERIFY(ReadOffset >= EndOffset); //otherwise no WaitData were needed
-
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " wait for data done: " << " readOffset " << ReadOffset << " EndOffset " << EndOffset
- << " newEndOffset " << record.GetEndOffset() << " commitOffset " << CommittedOffset
- << " clientCommitOffset " << ClientCommitOffset << " cookie " << ev->Get()->Record.GetCookie()
- );
-
- EndOffset = record.GetEndOffset();
- SizeLag = record.GetSizeLag();
-
- if (ReadOffset < EndOffset) {
- WaitForData = false;
- WaitDataInfly.clear();
- ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReady(Topic, Partition, WTime, SizeLag, ReadOffset, EndOffset));
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " ready for read with readOffset " << ReadOffset << " endOffset " << EndOffset
- );
- } else {
- if (PipeClient)
- WaitDataInPartition(ctx);
- }
- CheckRelease(ctx); //just for logging purpose
-}
-
-
-void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& ctx) {
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " READ FROM " << Topic->GetPrintableString() << " partition:" << Partition
- << " event " << ev->Get()->Request << " readOffset " << ReadOffset
- << " EndOffset " << EndOffset << " ClientCommitOffset " << ClientCommitOffset
- << " committedOffset " << CommittedOffset << " Guid " << ev->Get()->Guid
- );
-
- Y_VERIFY(!NeedRelease);
- Y_VERIFY(!Released);
-
- Y_VERIFY(ReadGuid.empty());
- Y_VERIFY(!RequestInfly);
-
- ReadGuid = ev->Get()->Guid;
-
- const auto& req = ev->Get()->Request.GetRead();
-
- NKikimrClient::TPersQueueRequest request;
-
- request.MutablePartitionRequest()->SetTopic(Topic->GetClientsideName());
-
- request.MutablePartitionRequest()->SetPartition(Partition);
- request.MutablePartitionRequest()->SetCookie((ui64)ReadOffset);
-
- ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
- auto read = request.MutablePartitionRequest()->MutableCmdRead();
- read->SetClientId(InternalClientId);
- read->SetClientDC(ClientDC);
- if (req.GetMaxCount()) {
- read->SetCount(req.GetMaxCount());
- }
- if (req.GetMaxSize()) {
- read->SetBytes(req.GetMaxSize());
- }
- if (req.GetMaxTimeLagMs()) {
- read->SetMaxTimeLagMs(req.GetMaxTimeLagMs());
- }
- if (req.GetReadTimestampMs()) {
- read->SetReadTimestampMs(req.GetReadTimestampMs());
- }
-
- read->SetOffset(ReadOffset);
- read->SetTimeoutMs(READ_TIMEOUT_DURATION.MilliSeconds());
- RequestInfly = true;
- CurrentRequest = request;
-
- if (!PipeClient) //Pipe will be recreated soon
- return;
-
- TAutoPtr<TEvPersQueue::TEvRequest> event(new TEvPersQueue::TEvRequest);
- event->Record.Swap(&request);
-
- NTabletPipe::SendData(ctx, PipeClient, event.Release());
-}
-
-
-void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContext& ctx) {
- const ui64 readId = ev->Get()->ReadId;
- const ui64 offset = ev->Get()->Offset;
- Y_VERIFY(offset != Max<ui64>()); // has concreete offset
- if (offset < ClientCommitOffset) {
- LOG_ERROR_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " commit done to too small position " << offset
- << " committedOffset " << ClientCommitOffset << " cookie " << readId
- );
- }
- Y_VERIFY(offset >= ClientCommitOffset);
-
- const bool hasProgress = offset > ClientCommitOffset;
-
- if (!hasProgress) {//nothing to commit for this partition
- if (CommitsInfly.empty()) {
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition
- << " commit done with no effect with cookie " << readId
- );
- ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition));
- CheckRelease(ctx);
- } else {
- CommitsInfly.push_back(std::pair<ui64, ui64>(readId, Max<ui64>()));
- }
- return;
- }
-
- ClientCommitOffset = offset;
- CommitsInfly.push_back(std::pair<ui64, ui64>(readId, offset));
-
- if (PipeClient) //if not then pipe will be recreated soon and SendCommit will be done
- SendCommit(readId, offset, ctx);
-}
-
-
-void TPartitionActor::Die(const TActorContext& ctx) {
- if (PipeClient)
- NTabletPipe::CloseClient(ctx, PipeClient);
- TActorBootstrapped<TPartitionActor>::Die(ctx);
-}
-
-void TPartitionActor::HandlePoison(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) {
- if (LockCounted)
- Counters.PartitionsToBeLocked.Dec();
- Die(ctx);
-}
-
-void TPartitionActor::Handle(TEvPQProxy::TEvDeadlineExceeded::TPtr& ev, const TActorContext& ctx) {
-
- WaitDataInfly.erase(ev->Get()->Cookie);
- if (ReadOffset >= EndOffset && WaitDataInfly.size() <= 1 && PipeClient) {
- Y_VERIFY(WaitForData);
- WaitDataInPartition(ctx);
- }
-
-}
-
-void TPartitionActor::HandleWakeup(const TActorContext& ctx) {
- if (ReadOffset >= EndOffset && WaitDataInfly.size() <= 1 && PipeClient) {
- Y_VERIFY(WaitForData);
- WaitDataInPartition(ctx);
- }
-}
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_session.h b/kikimr/yndx/grpc_services/persqueue/grpc_pq_session.h
deleted file mode 100644
index 22e2b61e5b..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_session.h
+++ /dev/null
@@ -1,317 +0,0 @@
-#pragma once
-
-#include "ydb/core/client/server/grpc_base.h"
-#include <library/cpp/grpc/server/grpc_server.h>
-#include <util/generic/queue.h>
-
-using grpc::Status;
-
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-///////////////////////////////////////////////////////////////////////////////
-
-using namespace NKikimrClient;
-
-template<class TResponse>
-class ISessionHandler : public TAtomicRefCount<ISessionHandler<TResponse>> {
-public:
- virtual ~ISessionHandler()
- { }
-
- /// Finish session.
- virtual void Finish() = 0;
-
- /// Send reply to client.
- virtual void Reply(const TResponse& resp) = 0;
-
- virtual void ReadyForNextRead() = 0;
-
- virtual bool IsShuttingDown() const = 0;
-};
-
-template<class TResponse>
-using ISessionHandlerRef = TIntrusivePtr<ISessionHandler<TResponse>>;
-
-
-template <class TRequest, class TResponse>
-class ISession : public ISessionHandler<TResponse>
-{
-
- using ISessionRef = TIntrusivePtr<ISession<TRequest, TResponse>>;
-
-protected:
- class TRequestCreated : public NGrpc::IQueueEvent {
- public:
- TRequestCreated(ISessionRef session)
- : Session(session)
- { }
-
- bool Execute(bool ok) override {
- if (!ok) {
- Session->DestroyStream("waiting stream creating failed");
- return false;
- }
-
- Session->OnCreated();
- return false;
- }
-
- void DestroyRequest() override {
- if (!Session->Context.c_call() && Session->ClientDone) {
- // AsyncNotifyWhenDone will not appear on the queue.
- delete Session->ClientDone;
- Session->ClientDone = nullptr;
- }
- delete this;
- }
-
- ISessionRef Session;
- };
-
- class TReadDone : public NGrpc::IQueueEvent {
- public:
- TReadDone(ISessionRef session)
- : Session(session)
- { }
-
- bool Execute(bool ok) override {
- if (ok) {
- Session->OnRead(Request);
- } else {
- if (Session->IsCancelled()) {
- Session->DestroyStream("reading from stream failed");
- } else {
- Session->OnDone();
- }
- }
- return false;
- }
-
- void DestroyRequest() override {
- delete this;
- }
-
- TRequest Request;
- ISessionRef Session;
- };
-
- class TWriteDone : public NGrpc::IQueueEvent {
- public:
- TWriteDone(ISessionRef session, ui64 size)
- : Session(session)
- , Size(size)
- { }
-
- bool Execute(bool ok) override {
- Session->OnWriteDone(Size);
- if (!ok) {
- Session->DestroyStream("writing to stream failed");
- return false;
- }
-
- TGuard<TSpinLock> lock(Session->Lock);
- if (Session->Responses.empty()) {
- Session->HaveWriteInflight = false;
- if (Session->NeedFinish) {
- lock.Release();
- Session->Stream.Finish(Status::OK, new TFinishDone(Session));
- }
- } else {
- auto resp = Session->Responses.front();
- Session->Responses.pop();
- lock.Release();
- ui64 sz = resp.ByteSize();
- Session->Stream.Write(resp, new TWriteDone(Session, sz));
- }
-
- return false;
- }
-
- void DestroyRequest() override {
- delete this;
- }
-
- ISessionRef Session;
- ui64 Size;
- };
-
- class TFinishDone : public NGrpc::IQueueEvent {
- public:
- TFinishDone(ISessionRef session)
- : Session(session)
- { }
-
- bool Execute(bool) override {
- Session->DestroyStream("some stream finished");
- return false;
- }
-
- void DestroyRequest() override {
- delete this;
- }
-
- ISessionRef Session;
- };
-
- class TClientDone : public NGrpc::IQueueEvent {
- public:
- TClientDone(ISessionRef session)
- : Session(session)
- {
- Session->ClientDone = this;
- }
-
- bool Execute(bool) override {
- Session->ClientIsDone = true;
- Session->DestroyStream("sesison closed");
- return false;
- }
-
- void DestroyRequest() override {
- Y_VERIFY(Session->ClientDone);
- Session->ClientDone = nullptr;
- delete this;
- }
-
- ISessionRef Session;
- };
-
-public:
- ISession(grpc::ServerCompletionQueue* cq)
- : CQ(cq)
- , Stream(&Context)
- , HaveWriteInflight(false)
- , NeedFinish(false)
- , ClientIsDone(false)
- {
- Context.AsyncNotifyWhenDone(new TClientDone(this));
- }
-
- TString GetDatabase() const {
- TString key = "x-ydb-database";
- const auto& clientMetadata = Context.client_metadata();
- const auto range = clientMetadata.equal_range(grpc::string_ref{key.data(), key.size()});
- if (range.first == range.second) {
- return "";
- }
-
- TVector<TStringBuf> values;
- values.reserve(std::distance(range.first, range.second));
-
- for (auto it = range.first; it != range.second; ++it) {
- return TString(it->second.data(), it->second.size());
- }
- return "";
- }
-
- TString GetPeerName() const {
- TString res(Context.peer());
- if (res.StartsWith("ipv4:[") || res.StartsWith("ipv6:[")) {
- size_t pos = res.find(']');
- Y_VERIFY(pos != TString::npos);
- res = res.substr(6, pos - 6);
- } else if (res.StartsWith("ipv4:")) {
- size_t pos = res.rfind(':');
- if (pos == TString::npos) {//no port
- res = res.substr(5);
- } else {
- res = res.substr(5, pos - 5);
- }
- } else {
- size_t pos = res.rfind(":"); //port
- if (pos != TString::npos) {
- res = res.substr(0, pos);
- }
- }
- return res;
- }
-
-protected:
-
- virtual void OnCreated() = 0;
- virtual void OnRead(const TRequest& request) = 0;
- virtual void OnDone() = 0;
- virtual void OnWriteDone(ui64 size) = 0;
-
- virtual void DestroyStream(const TString& reason, NPersQueue::NErrorCode::EErrorCode code = NPersQueue::NErrorCode::BAD_REQUEST) = 0;
-
- /// Start accepting session's requests.
- virtual void Start() = 0;
-
- bool IsCancelled() const {
- return ClientIsDone && Context.IsCancelled();
- }
-
- void ReplyWithError(const TString& description, NPersQueue::NErrorCode::EErrorCode code)
- {
- TResponse response;
- response.MutableError()->SetDescription(description);
- response.MutableError()->SetCode(code);
- Reply(response);
- Finish();
- }
-
- /// Finish session.
- void Finish() override {
- {
- TGuard<TSpinLock> lock(Lock);
- if (NeedFinish)
- return;
- if (HaveWriteInflight || !Responses.empty()) {
- NeedFinish = true;
- return;
- }
- HaveWriteInflight = true;
- }
-
- Stream.Finish(Status::OK, new TFinishDone(this));
- }
-
- /// Send reply to client.
- void Reply(const TResponse& resp) override {
- {
- TGuard<TSpinLock> lock(Lock);
- if (NeedFinish) //ignore responses after finish
- return;
- if (HaveWriteInflight || !Responses.empty()) {
- Responses.push(resp);
- return;
- } else {
- HaveWriteInflight = true;
- }
- }
-
- ui64 size = resp.ByteSize();
- Stream.Write(resp, new TWriteDone(this, size));
- }
-
- void ReadyForNextRead() override {
- {
- TGuard<TSpinLock> lock(Lock);
- if (NeedFinish) {
- return;
- }
- }
-
- auto read = new TReadDone(this);
- Stream.Read(&read->Request, read);
- }
-
-protected:
- grpc::ServerCompletionQueue* const CQ;
- grpc::ServerContext Context;
- grpc::ServerAsyncReaderWriter<TResponse, TRequest>
- Stream;
-private:
- TSpinLock Lock;
- bool HaveWriteInflight;
- bool NeedFinish;
- std::atomic<bool> ClientIsDone;
- TClientDone* ClientDone;
- TQueue<TResponse> Responses; //TODO: if Responses total size is too big - fail this session;
-};
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.cpp b/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.cpp
deleted file mode 100644
index 36ba3fa8f6..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-#include "grpc_pq_write.h"
-#include "grpc_pq_actor.h"
-#include "grpc_pq_session.h"
-#include "ydb/core/client/server/grpc_proxy_status.h"
-
-#include <ydb/core/base/appdata.h>
-#include <util/generic/queue.h>
-
-using namespace NActors;
-using namespace NKikimrClient;
-
-using grpc::Status;
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-using namespace NPersQueue;
-
-///////////////////////////////////////////////////////////////////////////////
-
-
-void TPQWriteServiceImpl::TSession::OnCreated() { // Start waiting for new session.
- Proxy->WaitWriteSession();
- if (Proxy->TooMuchSessions()) {
- ReplyWithError("proxy overloaded", NPersQueue::NErrorCode::OVERLOAD);
- return;
- }
- TMaybe<TString> localCluster = Proxy->AvailableLocalCluster();
- if (NeedDiscoverClusters) {
- if (!localCluster.Defined()) {
- ReplyWithError("initializing", NPersQueue::NErrorCode::INITIALIZING);
- return;
- } else if (localCluster->empty()) {
- ReplyWithError("cluster disabled", NPersQueue::NErrorCode::CLUSTER_DISABLED);
- return;
- } else {
- CreateActor(*localCluster);
- }
- } else {
- CreateActor(TString());
- }
- ReadyForNextRead();
-}
-
-void TPQWriteServiceImpl::TSession::OnRead(const TWriteRequest& request) {
-
- switch (request.GetRequestCase()) {
- case TWriteRequest::kInit: {
- SendEvent(new TEvPQProxy::TEvWriteInit(request, GetPeerName(), GetDatabase()));
- break;
- }
- case TWriteRequest::kDataBatch:
- case TWriteRequest::kData: {
- SendEvent(new TEvPQProxy::TEvWrite(request));
- break;
- }
- default: {
- ReplyWithError("unsupported request", NPersQueue::NErrorCode::BAD_REQUEST);
- }
- }
-}
-
-void TPQWriteServiceImpl::TSession::OnDone() {
- SendEvent(new TEvPQProxy::TEvDone());
-}
-
-TPQWriteServiceImpl::TSession::TSession(std::shared_ptr<TPQWriteServiceImpl> proxy,
- grpc::ServerCompletionQueue* cq, ui64 cookie, const TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, bool needDiscoverClusters)
- : ISession(cq)
- , Proxy(proxy)
- , Cookie(cookie)
- , SchemeCache(schemeCache)
- , Counters(counters)
- , NeedDiscoverClusters(needDiscoverClusters)
-{
-}
-
-void TPQWriteServiceImpl::TSession::Start() {
- if (!Proxy->IsShuttingDown()) {
- Proxy->RequestSession(&Context, &Stream, CQ, CQ, new TRequestCreated(this));
- }
-}
-
-ui64 TPQWriteServiceImpl::TSession::GetCookie() const {
- return Cookie;
-}
-
-void TPQWriteServiceImpl::TSession::DestroyStream(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) {
- // Send poison pill to the actor(if it is alive)
- SendEvent(new TEvPQProxy::TEvDieCommand("write-session " + ToString<ui64>(Cookie) + ": " + reason, errorCode));
- // Remove reference to session from "cookie -> session" map.
- Proxy->ReleaseSession(this);
-}
-
-bool TPQWriteServiceImpl::TSession::IsShuttingDown() const {
- return Proxy->IsShuttingDown();
-}
-
-void TPQWriteServiceImpl::TSession::CreateActor(const TString &localCluster) {
-
- auto classifier = Proxy->GetClassifier();
- ActorId = Proxy->ActorSystem->Register(
- new TWriteSessionActor(this, Cookie, SchemeCache, Counters, localCluster,
- classifier ? classifier->ClassifyAddress(GetPeerName())
- : "unknown"), TMailboxType::Simple, 0
- );
-}
-
-void TPQWriteServiceImpl::TSession::SendEvent(IEventBase* ev) {
- Proxy->ActorSystem->Send(ActorId, ev);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-
-TPQWriteServiceImpl::TPQWriteServiceImpl(grpc::ServerCompletionQueue* cq,
- NActors::TActorSystem* as, const TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const ui32 maxSessions)
- : CQ(cq)
- , ActorSystem(as)
- , SchemeCache(schemeCache)
- , Counters(counters)
- , MaxSessions(maxSessions)
- , NeedDiscoverClusters(false)
-{
-}
-
-void TPQWriteServiceImpl::InitClustersUpdater()
-{
- TAppData* appData = ActorSystem->AppData<TAppData>();
- NeedDiscoverClusters = !appData->PQConfig.GetTopicsAreFirstClassCitizen();
- if (NeedDiscoverClusters) {
- ActorSystem->Register(new TClustersUpdater(this));
- }
-}
-
-
-ui64 TPQWriteServiceImpl::NextCookie() {
- return AtomicIncrement(LastCookie);
-}
-
-
-void TPQWriteServiceImpl::ReleaseSession(TSessionRef session) {
- with_lock (Lock) {
- bool erased = Sessions.erase(session->GetCookie());
- if (erased) {
- ActorSystem->Send(MakeGRpcProxyStatusID(ActorSystem->NodeId), new TEvGRpcProxyStatus::TEvUpdateStatus(0, 0, -1, 0));
- }
- }
-}
-
-
-void TPQWriteServiceImpl::SetupIncomingRequests() {
- WaitWriteSession();
-}
-
-
-void TPQWriteServiceImpl::WaitWriteSession() {
-
- const ui64 cookie = NextCookie();
-
- ActorSystem->Send(MakeGRpcProxyStatusID(ActorSystem->NodeId), new TEvGRpcProxyStatus::TEvUpdateStatus(0,0,1,0));
-
- TSessionRef session(new TSession(shared_from_this(), CQ, cookie, SchemeCache, Counters, NeedDiscoverClusters));
- {
- with_lock (Lock) {
- Sessions[cookie] = session;
- }
- }
-
- session->Start();
-}
-
-
-bool TPQWriteServiceImpl::TooMuchSessions() {
- with_lock (Lock) {
- return Sessions.size() >= MaxSessions;
- }
-}
-
-
-TMaybe<TString> TPQWriteServiceImpl::AvailableLocalCluster() {
- with_lock (Lock) {
- return AvailableLocalClusterName;
- }
-}
-
-
-void TPQWriteServiceImpl::NetClassifierUpdated(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier) {
- auto g(Guard(Lock));
- if (!DatacenterClassifier) {
- for (auto it = Sessions.begin(); it != Sessions.end();) {
- auto jt = it++;
- jt->second->DestroyStream("datacenter classifier initialized, restart session please", NPersQueue::NErrorCode::INITIALIZING);
- }
- }
-
- DatacenterClassifier = classifier;
-}
-
-
-
-void TPQWriteServiceImpl::CheckClusterChange(const TString &localCluster, const bool enabled) {
- with_lock (Lock) {
- AvailableLocalClusterName = enabled ? localCluster : TString();
-
- if (!enabled) {
- for (auto it = Sessions.begin(); it != Sessions.end();) {
- auto jt = it++;
- jt->second->DestroyStream("cluster disabled", NPersQueue::NErrorCode::CLUSTER_DISABLED);
- }
- }
- }
-}
-
-
-///////////////////////////////////////////////////////////////////////////////
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.h b/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.h
deleted file mode 100644
index 40a0d64e47..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write.h
+++ /dev/null
@@ -1,148 +0,0 @@
-#pragma once
-
-#include "grpc_pq_clusters_updater_actor.h"
-#include "grpc_pq_session.h"
-
-#include <ydb/core/client/server/grpc_base.h>
-
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <library/cpp/actors/core/actorsystem.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/maybe.h>
-#include <util/system/mutex.h>
-
-namespace NKikimr {
-namespace NGRpcProxy {
-
-// Класс, отвечающий за обработку запросов на запись.
-
-class TPQWriteServiceImpl : public IPQClustersUpdaterCallback, public std::enable_shared_from_this<TPQWriteServiceImpl> {
-
- class TSession : public ISession<NPersQueue::TWriteRequest, NPersQueue::TWriteResponse>
- {
-
- void OnCreated() override;
- void OnRead(const NPersQueue::TWriteRequest& request) override;
- void OnDone() override;
- void OnWriteDone(ui64) override {};
-
- public:
- TSession(std::shared_ptr<TPQWriteServiceImpl> proxy,
- grpc::ServerCompletionQueue* cq, ui64 cookie, const TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, bool needDiscoverClusters);
- void Start() override;
- ui64 GetCookie() const;
- void DestroyStream(const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) override;
- bool IsShuttingDown() const override;
-
- private:
- void CreateActor(const TString& localCluster);
- void SendEvent(NActors::IEventBase* ev);
-
- private:
- std::shared_ptr<TPQWriteServiceImpl> Proxy;
- const ui64 Cookie;
-
- NActors::TActorId ActorId;
-
- const NActors::TActorId SchemeCache;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
-
- bool NeedDiscoverClusters;
- };
- using TSessionRef = TIntrusivePtr<TSession>;
-
-public:
- TPQWriteServiceImpl(grpc::ServerCompletionQueue* cq,
- NActors::TActorSystem* as, const NActors::TActorId& schemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const ui32 maxSessions);
- virtual ~TPQWriteServiceImpl() = default;
-
- void SetupIncomingRequests();
-
- virtual void RequestSession(::grpc::ServerContext* context, ::grpc::ServerAsyncReaderWriter< ::NPersQueue::TWriteResponse, ::NPersQueue::TWriteRequest>* stream,
- ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) = 0;
-
- void StopService() {
- AtomicSet(ShuttingDown_, 1);
- }
-
- bool IsShuttingDown() const {
- return AtomicGet(ShuttingDown_);
- }
- void InitClustersUpdater();
-
-private:
- ui64 NextCookie();
-
- //! Unregistry session object.
- void ReleaseSession(TSessionRef session);
-
- //! Start listening for incoming connections.
- void WaitWriteSession();
- bool TooMuchSessions();
- TMaybe<TString> AvailableLocalCluster();
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr GetClassifier() const {
- auto g(Guard(Lock));
- return DatacenterClassifier;
- }
- void CheckClusterChange(const TString& localCluster, const bool enabled) override;
- void NetClassifierUpdated(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier) override;
-
-private:
- grpc::ServerContext Context;
- grpc::ServerCompletionQueue* CQ;
-
- NActors::TActorSystem* ActorSystem;
- NActors::TActorId SchemeCache;
-
- TAtomic LastCookie = 0;
-
- TMutex Lock;
- THashMap<ui64, TSessionRef> Sessions;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
-
- ui32 MaxSessions;
- TMaybe<TString> AvailableLocalClusterName;
- TString SelectSourceIdQuery;
- TString UpdateSourceIdQuery;
- TString DeleteSourceIdQuery;
-
- TAtomic ShuttingDown_ = 0;
-
- bool NeedDiscoverClusters; // Legacy mode OR account-mode in multi-cluster setup;
-
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
-};
-
-
-class TPQWriteService : public TPQWriteServiceImpl {
-public:
- TPQWriteService(NPersQueue::PersQueueService::AsyncService* service,
- grpc::ServerCompletionQueue* cq,
- NActors::TActorSystem* as, const NActors::TActorId& schemeCache, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- const ui32 maxSessions)
- : TPQWriteServiceImpl(cq, as, schemeCache, counters, maxSessions)
- , Service(service)
- {}
-
- virtual ~TPQWriteService()
- {}
-
- void RequestSession(::grpc::ServerContext* context, ::grpc::ServerAsyncReaderWriter< ::NPersQueue::TWriteResponse, ::NPersQueue::TWriteRequest>* stream,
- ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) override
- {
- Service->RequestWriteSession(context, stream, new_call_cq, notification_cq, tag);
- }
-
-private:
- NPersQueue::PersQueueService::AsyncService* Service;
-};
-
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write_actor.cpp b/kikimr/yndx/grpc_services/persqueue/grpc_pq_write_actor.cpp
deleted file mode 100644
index ae2c3198c0..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/grpc_pq_write_actor.cpp
+++ /dev/null
@@ -1,1055 +0,0 @@
-#include "grpc_pq_actor.h"
-
-#include <ydb/core/persqueue/pq_database.h>
-#include <ydb/core/persqueue/write_meta.h>
-#include <ydb/core/protos/services.pb.h>
-#include <ydb/public/lib/deprecated/kicli/kicli.h>
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <ydb/library/persqueue/topic_parser/counters.h>
-#include <ydb/services/lib/sharding/sharding.h>
-
-#include <library/cpp/actors/core/log.h>
-#include <library/cpp/digest/md5/md5.h>
-#include <util/string/hex.h>
-#include <util/string/vector.h>
-#include <util/string/escape.h>
-
-using namespace NActors;
-using namespace NKikimrClient;
-
-
-namespace NKikimr {
-using namespace NMsgBusProxy::NPqMetaCacheV2;
-using namespace NSchemeCache;
-using namespace NPQ;
-
-template <>
-void FillChunkDataFromReq(NKikimrPQClient::TDataChunk& proto, const NPersQueue::TWriteRequest::TData& data) {
- proto.SetData(data.GetData());
- proto.SetSeqNo(data.GetSeqNo());
- proto.SetCreateTime(data.GetCreateTimeMs());
- proto.SetCodec(data.GetCodec());
-}
-
-template <>
-void FillExtraFieldsForDataChunk(
- const NPersQueue::TWriteRequest::TInit& init,
- NKikimrPQClient::TDataChunk& data,
- TString& server,
- TString& ident,
- TString& logType,
- TString& file
-) {
- for (ui32 i = 0; i < init.GetExtraFields().ItemsSize(); ++i) {
- const auto& item = init.GetExtraFields().GetItems(i);
- if (item.GetKey() == "server") {
- server = item.GetValue();
- } else if (item.GetKey() == "ident") {
- ident = item.GetValue();
- } else if (item.GetKey() == "logtype") {
- logType = item.GetValue();
- } else if (item.GetKey() == "file") {
- file = item.GetValue();
- } else {
- auto res = data.MutableExtraFields()->AddItems();
- res->SetKey(item.GetKey());
- res->SetValue(item.GetValue());
- }
- }
-}
-
-namespace NGRpcProxy {
-
-using namespace NPersQueue;
-
-static const ui32 MAX_RESERVE_REQUESTS_INFLIGHT = 5;
-
-static const ui32 MAX_BYTES_INFLIGHT = 1_MB;
-static const TDuration SOURCEID_UPDATE_PERIOD = TDuration::Hours(1);
-
-//TODO: add here tracking of bytes in/out
-
-TWriteSessionActor::TWriteSessionActor(IWriteSessionHandlerRef handler, const ui64 cookie, const TActorId& schemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TString& localDC,
- const TMaybe<TString> clientDC)
- : Handler(handler)
- , State(ES_CREATED)
- , SchemeCache(schemeCache)
- , PeerName("")
- , Cookie(cookie)
- , Partition(0)
- , PreferedPartition(Max<ui32>())
- , NumReserveBytesRequests(0)
- , WritesDone(false)
- , Counters(counters)
- , BytesInflight_(0)
- , BytesInflightTotal_(0)
- , NextRequestInited(false)
- , NextRequestCookie(0)
- , Token(nullptr)
- , ACLCheckInProgress(true)
- , FirstACLCheck(true)
- , ForceACLCheck(false)
- , RequestNotChecked(true)
- , LastACLCheckTimestamp(TInstant::Zero())
- , LogSessionDeadline(TInstant::Zero())
- , BalancerTabletId(0)
- , PipeToBalancer()
- , LocalDC(localDC)
- , ClientDC(clientDC ? *clientDC : "other")
- , LastSourceIdUpdate(TInstant::Zero())
- , SourceIdCreateTime(0)
- , SourceIdUpdatesInflight(0)
-
-{
- Y_ASSERT(Handler);
-}
-
-TWriteSessionActor::~TWriteSessionActor() = default;
-
-
-void TWriteSessionActor::Bootstrap(const TActorContext& ctx) {
- if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- ++(*GetServiceCounters(Counters, "pqproxy|writeSession")->GetCounter("SessionsCreatedTotal", true));
- }
- Become(&TThis::StateFunc);
-
- Database = NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig);
- const auto& root = AppData(ctx)->PQConfig.GetRoot();
- SelectSourceIdQuery = GetSourceIdSelectQuery(root);
- UpdateSourceIdQuery = GetUpdateIdSelectQuery(root);
- ConverterFactory = MakeHolder<NPersQueue::TTopicNamesConverterFactory>(
- AppData(ctx)->PQConfig, LocalDC
- );
- StartTime = ctx.Now();
-}
-
-
-void TWriteSessionActor::Die(const TActorContext& ctx) {
- if (State == ES_DYING)
- return;
- State = ES_DYING;
- if (Writer)
- ctx.Send(Writer, new TEvents::TEvPoisonPill());
-
- if (PipeToBalancer)
- NTabletPipe::CloseClient(ctx, PipeToBalancer);
-
- if (SessionsActive) {
- SessionsActive.Dec();
- BytesInflight.Dec(BytesInflight_);
- BytesInflightTotal.Dec(BytesInflightTotal_);
- }
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " is DEAD");
-
- if (!Handler->IsShuttingDown())
- Handler->Finish();
- TActorBootstrapped<TWriteSessionActor>::Die(ctx);
-}
-
-void TWriteSessionActor::CheckFinish(const TActorContext& ctx) {
- if (!WritesDone)
- return;
- if (State != ES_INITED) {
- CloseSession("out of order Writes done before initialization", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- if (Writes.empty() && FormedWrites.empty() && SentMessages.empty()) {
- CloseSession("", NPersQueue::NErrorCode::OK, ctx);
- return;
- }
-}
-
-void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& ctx) {
- WritesDone = true;
- CheckFinish(ctx);
-}
-
-void TWriteSessionActor::CheckACL(const TActorContext& ctx) {
- Y_VERIFY(ACLCheckInProgress);
- Y_VERIFY(SecurityObject);
- NACLib::EAccessRights rights = NACLib::EAccessRights::UpdateRow;
- if (!AppData(ctx)->PQConfig.GetCheckACL() || SecurityObject->CheckAccess(rights, *Token)) {
- ACLCheckInProgress = false;
- if (FirstACLCheck) {
- FirstACLCheck = false;
- DiscoverPartition(ctx);
- }
- } else {
- TString errorReason = Sprintf("access to topic '%s' denied for '%s' due to 'no WriteTopic rights', Marker# PQ1125",
- DiscoveryConverter->GetPrintableString().c_str(),
- Token->GetUserSID().c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::ACCESS_DENIED, ctx);
- }
-}
-
-void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActorContext& ctx) {
- THolder<TEvPQProxy::TEvWriteInit> event(ev->Release());
-
- if (State != ES_CREATED) {
- //answer error
- CloseSession("got second init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- const auto& init = event->Request.GetInit();
-
- if (init.GetTopic().empty() || init.GetSourceId().empty()) {
- CloseSession("no topic or SourceId in init request", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- if (init.GetProxyCookie() != ctx.SelfID.NodeId() && init.GetProxyCookie() != MAGIC_COOKIE_VALUE) {
- CloseSession("you must perform ChooseProxy request at first and go to ProxyName server with ProxyCookie", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- //1. Database - !(Root or empty) (Need to bring root DB(s) list to PQConfig) - ONLY search modern path /Database/Path
- //2. No database. Try parse and resolve account to database. If possible, try search this path.
- //3. Fallback from 2 - legacy mode.
-
- DiscoveryConverter = ConverterFactory->MakeDiscoveryConverter(init.GetTopic(), true, LocalDC, Database);
- if (!DiscoveryConverter->IsValid()) {
- CloseSession(
- TStringBuilder() << "incorrect topic \"" << DiscoveryConverter->GetOriginalTopic()
- << "\": " << DiscoveryConverter->GetReason(),
- NPersQueue::NErrorCode::BAD_REQUEST,
- ctx
- );
- }
- PeerName = event->PeerName;
- if (!event->Database.empty()) {
- Database = event->Database;
- }
-
- SourceId = init.GetSourceId();
- //TODO: check that sourceId does not have characters '"\_% - espace them on client may be?
-
- Auth = event->Request.GetCredentials();
- event->Request.ClearCredentials();
- Y_PROTOBUF_SUPPRESS_NODISCARD Auth.SerializeToString(&AuthStr);
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << " from " << PeerName);
- UserAgent = init.GetVersion();
- LogSession(ctx);
-
- auto* request = new TEvDescribeTopicsRequest({DiscoveryConverter});
- //TODO: GetNode for /Root/PQ then describe from balancer
- ctx.Send(SchemeCache, request);
- State = ES_WAIT_SCHEME_2;
- InitRequest = init;
- PreferedPartition = init.GetPartitionGroup() > 0 ? init.GetPartitionGroup() - 1 : Max<ui32>();
-}
-
-void TWriteSessionActor::InitAfterDiscovery(const TActorContext& ctx) {
- try {
- EncodedSourceId = NSourceIdEncoding::EncodeSrcId(FullConverter->GetTopicForSrcIdHash(), SourceId);
- } catch (yexception& e) {
- CloseSession(TStringBuilder() << "incorrect sourceId \"" << SourceId << "\": " << e.what(), NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- InitMeta = GetInitialDataChunk(InitRequest, FullConverter->GetClientsideName(), PeerName);
-
- auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
- Aggr = {{{{"Account", FullConverter->GetAccount()}}, {"total"}}};
-
- SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
- SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
- SLITotal.Inc();
-}
-
-
-void TWriteSessionActor::SetupCounters()
-{
- if (SessionsCreated) {
- return;
- }
-
- //now topic is checked, can create group for real topic, not garbage
- auto subGroup = GetServiceCounters(Counters, "pqproxy|writeSession");
- Y_VERIFY(FullConverter);
- auto aggr = GetLabels(FullConverter);
-
- BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false);
- SessionsWithoutAuth = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WithoutAuth"}, true);
- BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false);
- SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsCreated"}, true);
- SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"SessionsActive"}, false);
- Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"Errors"}, true);
-
- SessionsCreated.Inc();
- SessionsActive.Inc();
-}
-
-
-void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& dbId, const TString& folderId)
-{
- if (SessionsCreated) {
- return;
- }
-
- //now topic is checked, can create group for real topic, not garbage
- auto subGroup = GetCountersForStream(Counters);
- Y_VERIFY(FullConverter);
- auto aggr = GetLabelsForStream(FullConverter, cloudId, dbId, folderId);
-
- BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false, "name");
- SessionsWithoutAuth = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_without_auth"}, true, "name");
- BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false, "name");
- SessionsCreated = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_created_per_second"}, true, "name");
- SessionsActive = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.sessions_active"}, false, "name");
- Errors = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.errors_per_second"}, true, "name");
-
- SessionsCreated.Inc();
- SessionsActive.Inc();
-}
-
-
-void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(State == ES_WAIT_SCHEME_2);
- auto& res = ev->Get()->Result;
- Y_VERIFY(res->ResultSet.size() == 1);
-
- auto& entry = res->ResultSet[0];
- TString errorReason;
-
- auto& path = entry.Path;
- auto& topic = ev->Get()->TopicsRequested[0];
- switch (entry.Status) {
- case TSchemeCacheNavigate::EStatus::RootUnknown: {
- errorReason = Sprintf("path '%s' has incorrect root prefix, Marker# PQ14", JoinPath(path).c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx);
- return;
- }
- case TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
- errorReason = Sprintf("no path '%s', Marker# PQ151", JoinPath(path).c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx);
- return;
- }
- case TSchemeCacheNavigate::EStatus::Ok:
- break;
- default: {
- errorReason = Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1", path.back().c_str(),
- ToString(entry.Status).c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::ERROR, ctx);
- break;
- }
- }
- if (!entry.PQGroupInfo) {
-
- errorReason = Sprintf("topic '%s' describe error, reason - could not retrieve topic metadata, Marker# PQ99",
- topic->GetPrintableString().c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- PQInfo = entry.PQGroupInfo;
- const auto& description = PQInfo->Description;
- //const TString topicName = description.GetName();
-
- if (entry.Kind != TSchemeCacheNavigate::EKind::KindTopic) {
- errorReason = Sprintf("item '%s' is not a topic, Marker# PQ13", DiscoveryConverter->GetPrintableString().c_str());
- CloseSession(errorReason, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- FullConverter = DiscoveryConverter->UpgradeToFullConverter(description.GetPQTabletConfig());
- InitAfterDiscovery(ctx);
- SecurityObject = entry.SecurityObject;
-
- Y_VERIFY(description.PartitionsSize() > 0);
-
- for (ui32 i = 0; i < description.PartitionsSize(); ++i) {
- const auto& pi = description.GetPartitions(i);
- PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId();
- }
- BalancerTabletId = description.GetBalancerTabletID();
- DatabaseId = description.GetPQTabletConfig().GetYdbDatabaseId();
- FolderId = description.GetPQTabletConfig().GetYcFolderId();
-
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- const auto& tabletConfig = description.GetPQTabletConfig();
- SetupCounters(tabletConfig.GetYcCloudId(), tabletConfig.GetYdbDatabaseId(),
- tabletConfig.GetYcFolderId());
- } else {
- SetupCounters();
- }
-
- if (!PipeToBalancer) {
- NTabletPipe::TClientConfig clientConfig;
- clientConfig.RetryPolicy = {
- .RetryLimitCount = 6,
- .MinRetryTime = TDuration::MilliSeconds(10),
- .MaxRetryTime = TDuration::MilliSeconds(100),
- .BackoffMultiplier = 2,
- .DoFirstRetryInstantly = true
- };
- PipeToBalancer = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, BalancerTabletId, clientConfig));
- }
-
- if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- //ACLCheckInProgress is still true - no recheck will be done
- LOG_WARN_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session without AuthInfo : " << DiscoveryConverter->GetPrintableString()
- << " sourceId " << SourceId << " from " << PeerName);
- SessionsWithoutAuth.Inc();
- if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
- CloseSession("Unauthenticated access is forbidden, please provide credentials", NPersQueue::NErrorCode::ACCESS_DENIED, ctx);
- return;
- }
- if (FirstACLCheck) {
- FirstACLCheck = false;
- DiscoverPartition(ctx);
- return;
- }
- }
-
- InitCheckACL(ctx);
-}
-
-void TWriteSessionActor::InitCheckACL(const TActorContext& ctx) {
-
- Y_VERIFY(ACLCheckInProgress);
-
- TString ticket;
- switch (Auth.GetCredentialsCase()) {
- case NPersQueueCommon::TCredentials::kTvmServiceTicket:
- ticket = Auth.GetTvmServiceTicket();
- break;
- case NPersQueueCommon::TCredentials::kOauthToken:
- ticket = Auth.GetOauthToken();
- break;
- default:
- CloseSession("Uknown Credentials case", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- auto entries = GetTicketParserEntries(DatabaseId, FolderId);
- ctx.Send(MakeTicketParserID(), new TEvTicketParser::TEvAuthorizeTicket({
- .Database = Database,
- .Ticket = ticket,
- .PeerName = PeerName,
- .Entries = entries
- }));
-}
-
-void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(ACLCheckInProgress);
- TString ticket = ev->Get()->Ticket;
- TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***";
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " << ev->Get()->Error << " user: "
- << (ev->Get()->Error.empty() ? ev->Get()->Token->GetUserSID() : ""));
-
- if (!ev->Get()->Error.empty()) {
- CloseSession(TStringBuilder() << "Ticket parsing error: " << ev->Get()->Error, NPersQueue::NErrorCode::ACCESS_DENIED, ctx);
- return;
- }
- Token = ev->Get()->Token;
-
-
- Y_VERIFY(ACLCheckInProgress);
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check");
- CheckACL(ctx);
-}
-
-void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) {
- Y_VERIFY(FullConverter);
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- auto partitionId = PreferedPartition;
- if (PreferedPartition == Max<ui32>()) {
- partitionId = NKikimr::NDataStreams::V1::ShardFromDecimal(
- NKikimr::NDataStreams::V1::HexBytesToDecimal(MD5::Calc(SourceId)), PartitionToTablet.size()
- );
- }
- ProceedPartition(partitionId, ctx);
- return;
- }
- //read from DS
- // Hash was always valid here, so new and old are the same
- //currently, Topic contains full primary path
- SendSelectPartitionRequest(EncodedSourceId.Hash, FullConverter->GetPrimaryPath(), ctx);
- //previously topic was like "rt3.dc--account--topic"
- SendSelectPartitionRequest(EncodedSourceId.Hash, FullConverter->GetTopicForSrcId(), ctx);
- State = ES_WAIT_TABLE_REQUEST_1;
-}
-
-void TWriteSessionActor::SendSelectPartitionRequest(ui32 hash, const TString &topic,
- const NActors::TActorContext &ctx
-) {
- auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
- ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
- ev->Record.MutableRequest()->SetKeepSession(false);
- ev->Record.MutableRequest()->SetQuery(SelectSourceIdQuery);
- ev->Record.MutableRequest()->SetDatabase(Database);
- ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
- ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
- ev->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
- NClient::TParameters parameters;
- parameters["$Hash"] = hash; // 'Valid' hash - short legacy name (account--topic)
- parameters["$Topic"] = topic; //currently, Topic contains full primary path
- parameters["$SourceId"] = EncodedSourceId.EscapedSourceId;
-
- ev->Record.MutableRequest()->MutableParameters()->Swap(&parameters);
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- SelectReqsInflight++;
-}
-
-
-void TWriteSessionActor::UpdatePartition(const TActorContext& ctx) {
- Y_VERIFY(State == ES_WAIT_TABLE_REQUEST_1 || State == ES_WAIT_NEXT_PARTITION);
- auto ev = MakeUpdateSourceIdMetadataRequest(EncodedSourceId.Hash, FullConverter->GetPrimaryPath()); // Now Topic is a path
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- SourceIdUpdatesInflight++;
-
- //Previously Topic contained legacy name with DC (rt3.dc1--acc--topic)
- ev = MakeUpdateSourceIdMetadataRequest(EncodedSourceId.Hash, FullConverter->GetTopicForSrcId());
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- SourceIdUpdatesInflight++;
-
- State = ES_WAIT_TABLE_REQUEST_2;
-}
-
-void TWriteSessionActor::RequestNextPartition(const TActorContext& ctx) {
- Y_VERIFY(State == ES_WAIT_TABLE_REQUEST_1);
- State = ES_WAIT_NEXT_PARTITION;
- THolder<TEvPersQueue::TEvGetPartitionIdForWrite> x(new TEvPersQueue::TEvGetPartitionIdForWrite);
- Y_VERIFY(PipeToBalancer);
-
- NTabletPipe::SendData(ctx, PipeToBalancer, x.Release());
-}
-
-void TWriteSessionActor::Handle(TEvPersQueue::TEvGetPartitionIdForWriteResponse::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(State == ES_WAIT_NEXT_PARTITION);
- Partition = ev->Get()->Record.GetPartitionId();
- UpdatePartition(ctx);
-}
-
-void TWriteSessionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TActorContext &ctx) {
- auto& record = ev->Get()->Record.GetRef();
-
- if (record.GetYdbStatus() == Ydb::StatusIds::ABORTED) {
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " sourceID "
- << SourceId << " escaped " << EncodedSourceId.EscapedSourceId << " discover partition race, retrying");
- DiscoverPartition(ctx);
- return;
- }
-
- if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) {
- TStringBuilder errorReason;
- errorReason << "internal error in kqp Marker# PQ50 : " << record;
- if (State == EState::ES_INITED) {
- LOG_WARN_S(ctx, NKikimrServices::PQ_WRITE_PROXY, errorReason);
- SourceIdUpdatesInflight--;
- } else {
- CloseSession(errorReason, NPersQueue::NErrorCode::ERROR, ctx);
- }
- return;
- }
-
- if (State == EState::ES_WAIT_TABLE_REQUEST_1) {
- SelectReqsInflight--;
- auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0);
-
- if (t.ListSize() != 0) {
- auto& tt = t.GetList(0).GetStruct(0);
- if (tt.HasOptional() && tt.GetOptional().HasUint32()) { //already got partition
- auto accessTime = t.GetList(0).GetStruct(2).GetOptional().GetUint64();
- if (accessTime > MaxSrcIdAccessTime) { // AccessTime
- Partition = tt.GetOptional().GetUint32();
- PartitionFound = true;
- SourceIdCreateTime = t.GetList(0).GetStruct(1).GetOptional().GetUint64();
- MaxSrcIdAccessTime = accessTime;
- }
- }
- }
- if (SelectReqsInflight != 0) {
- return;
- }
- if (SourceIdCreateTime == 0) {
- SourceIdCreateTime = TInstant::Now().MilliSeconds();
- }
- if (PartitionFound && PreferedPartition < Max<ui32>() && Partition != PreferedPartition) {
- CloseSession(TStringBuilder() << "SourceId " << SourceId << " is already bound to PartitionGroup " << (Partition + 1) << ", but client provided " << (PreferedPartition + 1) << ". SourceId->PartitionGroup binding cannot be changed, either use another SourceId, specify PartitionGroup " << (Partition + 1) << ", or do not specify PartitionGroup at all.",
- NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " sourceID "
- << SourceId << " escaped " << EncodedSourceId.EscapedSourceId << " hash " << EncodedSourceId.Hash << " partition " << Partition << " partitions "
- << PartitionToTablet.size() << "(" << EncodedSourceId.Hash % PartitionToTablet.size() << ") create " << SourceIdCreateTime << " result " << t);
-
- if (!PartitionFound && (PreferedPartition < Max<ui32>() || !AppData(ctx)->PQConfig.GetRoundRobinPartitionMapping())) {
- Partition = PreferedPartition < Max<ui32>() ? PreferedPartition : EncodedSourceId.Hash % PartitionToTablet.size(); //choose partition default value
- PartitionFound = true;
- }
-
- if (PartitionFound) {
- UpdatePartition(ctx);
- } else {
- RequestNextPartition(ctx);
- }
- return;
- } else if (State == EState::ES_WAIT_TABLE_REQUEST_2) {
- Y_VERIFY(SourceIdUpdatesInflight > 0);
- SourceIdUpdatesInflight--;
- if (SourceIdUpdatesInflight == 0) {
- LastSourceIdUpdate = ctx.Now();
- ProceedPartition(Partition, ctx);
- }
- } else if (State == EState::ES_INITED) {
- Y_VERIFY(SourceIdUpdatesInflight > 0);
- SourceIdUpdatesInflight--;
- if (SourceIdUpdatesInflight == 0) {
- LastSourceIdUpdate = ctx.Now();
- }
- } else {
- Y_FAIL("Wrong state");
- }
-}
-
-THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMetadataRequest(
- ui32 hash, const TString& topic
-) {
- auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
-
- ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
- ev->Record.MutableRequest()->SetQuery(UpdateSourceIdQuery);
- ev->Record.MutableRequest()->SetDatabase(Database);
- ev->Record.MutableRequest()->SetKeepSession(false);
- ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
- ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
- ev->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
-
- NClient::TParameters parameters;
- parameters["$Hash"] = hash;
- parameters["$Topic"] = topic; //Previously Topic contained legacy name with DC (rt3.dc1--acc--topic)
- parameters["$SourceId"] = EncodedSourceId.EscapedSourceId;
- parameters["$CreateTime"] = SourceIdCreateTime;
- parameters["$AccessTime"] = TInstant::Now().MilliSeconds();
- parameters["$Partition"] = Partition;
- ev->Record.MutableRequest()->MutableParameters()->Swap(&parameters);
-
- return ev;
-}
-
-
-void TWriteSessionActor::Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr &ev, const TActorContext &ctx) {
- auto& record = ev->Get()->Record;
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " sourceID "
- << SourceId << " escaped " << EncodedSourceId.EscapedSourceId << " discover partition error - " << record);
- CloseSession("Internal error on discovering partition", NPersQueue::NErrorCode::ERROR, ctx);
-}
-
-
-void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorContext& ctx) {
- Partition = partition;
- auto it = PartitionToTablet.find(Partition);
-
- ui64 tabletId = it != PartitionToTablet.end() ? it->second : 0;
-
- if (!tabletId) {
- CloseSession(
- Sprintf("no partition %u in topic '%s', Marker# PQ4", Partition, DiscoveryConverter->GetPrintableString().c_str()),
- NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx
- );
- return;
- }
-
- Writer = ctx.RegisterWithSameMailbox(NPQ::CreatePartitionWriter(ctx.SelfID, tabletId, Partition, SourceId));
- State = ES_WAIT_WRITER_INIT;
-
- ui32 border = AppData(ctx)->PQConfig.GetWriteInitLatencyBigMs();
- auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
-
- InitLatency = NKikimr::NPQ::CreateSLIDurationCounter(subGroup, Aggr, "WriteInit", border, {100, 200, 500, 1000, 1500, 2000, 5000, 10000, 30000, 99999999});
- SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsBigLatency"}, true, "sensor", false);
-
- ui32 initDurationMs = (ctx.Now() - StartTime).MilliSeconds();
- InitLatency.IncFor(initDurationMs, 1);
- if (initDurationMs >= border) {
- SLIBigLatency.Inc();
- }
-}
-
-void TWriteSessionActor::CloseSession(const TString& errorReason, const NPersQueue::NErrorCode::EErrorCode errorCode, const NActors::TActorContext& ctx) {
- if (errorCode != NPersQueue::NErrorCode::OK) {
- if (InternalErrorCode(errorCode)) {
- SLIErrors.Inc();
- }
-
- if (Errors) {
- Errors.Inc();
- } else if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) {
- ++(*GetServiceCounters(Counters, "pqproxy|writeSession")->GetCounter("Errors", true));
- }
-
- TWriteResponse result;
-
- auto error = result.MutableError();
- error->SetDescription(errorReason);
- error->SetCode(errorCode);
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session error cookie: " << Cookie << " reason: \"" << errorReason << "\" code: " << EErrorCode_Name(errorCode) << " sessionId: " << OwnerCookie);
-
- Handler->Reply(result);
- } else {
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session closed cookie: " << Cookie << " sessionId: " << OwnerCookie);
- }
-
- Die(ctx);
-}
-
-void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx) {
- if (State != ES_WAIT_WRITER_INIT) {
- return CloseSession("got init result but not wait for it", NPersQueue::NErrorCode::ERROR, ctx);
- }
-
- const auto& result = *ev->Get();
- if (!result.IsSuccess()) {
- const auto& error = result.GetError();
- if (error.Response.HasErrorCode()) {
- return CloseSession("status is not ok: " + error.Response.GetErrorReason(), error.Response.GetErrorCode(), ctx);
- } else {
- return CloseSession("error at writer init: " + error.Reason, NPersQueue::NErrorCode::ERROR, ctx);
- }
- }
-
- OwnerCookie = result.GetResult().OwnerCookie;
- const auto& maxSeqNo = result.GetResult().SourceIdInfo.GetSeqNo();
-
- TWriteResponse response;
- auto init = response.MutableInit();
- init->SetSessionId(EscapeC(OwnerCookie));
- init->SetMaxSeqNo(maxSeqNo);
- init->SetPartition(Partition);
- Y_VERIFY(FullConverter);
- init->SetTopic(FullConverter->GetClientsideName());
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition
- << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie);
-
- Handler->Reply(response);
-
- State = ES_INITED;
-
- ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup());
-
- //init completed; wait for first data chunk
- NextRequestInited = true;
- Handler->ReadyForNextRead();
-}
-
-void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteAccepted::TPtr& ev, const TActorContext& ctx) {
- if (State != ES_INITED) {
- return CloseSession("got write permission but not wait for it", NPersQueue::NErrorCode::ERROR, ctx);
- }
-
- Y_VERIFY(!FormedWrites.empty());
- TWriteRequestBatchInfo::TPtr writeRequest = std::move(FormedWrites.front());
-
- if (ev->Get()->Cookie != writeRequest->Cookie) {
- return CloseSession("out of order reserve bytes response from server, may be previous is lost", NPersQueue::NErrorCode::ERROR, ctx);
- }
-
- FormedWrites.pop_front();
-
- ui64 diff = writeRequest->ByteSize;
-
- SentMessages.emplace_back(std::move(writeRequest));
-
- BytesInflight_ -= diff;
- BytesInflight.Dec(diff);
-
- if (!NextRequestInited && BytesInflight_ < MAX_BYTES_INFLIGHT) { //allow only one big request to be readed but not sended
- NextRequestInited = true;
- Handler->ReadyForNextRead();
- }
-
- --NumReserveBytesRequests;
- if (!Writes.empty())
- GenerateNextWriteRequest(ctx);
-}
-
-void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvWriteResponse::TPtr& ev, const TActorContext& ctx) {
- if (State != ES_INITED) {
- return CloseSession("got write response but not wait for it", NPersQueue::NErrorCode::ERROR, ctx);
- }
-
- const auto& result = *ev->Get();
- if (!result.IsSuccess()) {
- const auto& record = result.Record;
- if (record.HasErrorCode()) {
- return CloseSession("status is not ok: " + record.GetErrorReason(), record.GetErrorCode(), ctx);
- } else {
- return CloseSession("error at write: " + result.GetError().Reason, NPersQueue::NErrorCode::ERROR, ctx);
- }
- }
-
- const auto& resp = result.Record.GetPartitionResponse();
-
- if (SentMessages.empty()) {
- CloseSession("got too many replies from server, internal error", NPersQueue::NErrorCode::ERROR, ctx);
- return;
- }
-
- TWriteRequestBatchInfo::TPtr writeRequest = std::move(SentMessages.front());
- SentMessages.pop_front();
-
- if (resp.GetCookie() != writeRequest->Cookie) {
- return CloseSession("out of order write response from server, may be previous is lost", NPersQueue::NErrorCode::ERROR, ctx);
- }
-
- auto addAck = [](const TPersQueuePartitionResponse::TCmdWriteResult& res, TWriteResponse::TAck* ack, TWriteResponse::TStat* stat) {
- ack->SetSeqNo(res.GetSeqNo());
- ack->SetOffset(res.GetOffset());
- ack->SetAlreadyWritten(res.GetAlreadyWritten());
-
- stat->SetTotalTimeInPartitionQueueMs(
- Max(res.GetTotalTimeInPartitionQueueMs(), stat->GetTotalTimeInPartitionQueueMs()));
- stat->SetPartitionQuotedTimeMs(
- Max(res.GetPartitionQuotedTimeMs(), stat->GetPartitionQuotedTimeMs()));
- stat->SetTopicQuotedTimeMs(
- Max(res.GetTopicQuotedTimeMs(), stat->GetTopicQuotedTimeMs()));
- stat->SetWriteTimeMs(
- Max(res.GetWriteTimeMs(), stat->GetWriteTimeMs()));
- };
-
- size_t cmdWriteResultIndex = 0;
- for (const auto& userWriteRequest : writeRequest->UserWriteRequests) {
- TWriteResponse result;
- if (userWriteRequest->Request.HasDataBatch()) {
- if (resp.CmdWriteResultSize() - cmdWriteResultIndex < userWriteRequest->Request.GetDataBatch().DataSize()) {
- CloseSession("too less responses from server", NPersQueue::NErrorCode::ERROR, ctx);
- return;
- }
- for (size_t endIndex = cmdWriteResultIndex + userWriteRequest->Request.GetDataBatch().DataSize(); cmdWriteResultIndex < endIndex; ++cmdWriteResultIndex) {
- addAck(resp.GetCmdWriteResult(cmdWriteResultIndex),
- result.MutableAckBatch()->AddAck(),
- result.MutableAckBatch()->MutableStat());
- }
- } else {
- Y_VERIFY(userWriteRequest->Request.HasData());
- if (cmdWriteResultIndex >= resp.CmdWriteResultSize()) {
- CloseSession("too less responses from server", NPersQueue::NErrorCode::ERROR, ctx);
- return;
- }
- auto* ack = result.MutableAck();
- addAck(resp.GetCmdWriteResult(cmdWriteResultIndex), ack, ack->MutableStat());
- ++cmdWriteResultIndex;
- }
- Handler->Reply(result);
- }
-
- ui64 diff = writeRequest->ByteSize;
-
- BytesInflightTotal_ -= diff;
- BytesInflightTotal.Dec(diff);
-
- CheckFinish(ctx);
-}
-
-void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvDisconnected::TPtr&, const TActorContext& ctx) {
- CloseSession("pipe to partition's tablet is dead", NPersQueue::NErrorCode::ERROR, ctx);
-}
-
-void TWriteSessionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) {
- TEvTabletPipe::TEvClientConnected *msg = ev->Get();
- if (msg->Status != NKikimrProto::OK) {
- CloseSession(TStringBuilder() << "pipe to tablet is dead " << msg->TabletId, NPersQueue::NErrorCode::ERROR, ctx);
- return;
- }
-}
-
-void TWriteSessionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
- CloseSession(TStringBuilder() << "pipe to tablet is dead " << ev->Get()->TabletId, NPersQueue::NErrorCode::ERROR, ctx);
-}
-
-void TWriteSessionActor::GenerateNextWriteRequest(const TActorContext& ctx) {
- TWriteRequestBatchInfo::TPtr writeRequest = new TWriteRequestBatchInfo();
-
- auto ev = MakeHolder<NPQ::TEvPartitionWriter::TEvWriteRequest>(++NextRequestCookie);
- NKikimrClient::TPersQueueRequest& request = ev->Record;
-
- writeRequest->UserWriteRequests = std::move(Writes);
- Writes.clear();
-
- i64 diff = 0;
- auto addData = [&](const TWriteRequest::TData& data) {
- auto w = request.MutablePartitionRequest()->AddCmdWrite();
- w->SetData(GetSerializedData(InitMeta, data));
- w->SetClientDC(ClientDC);
- w->SetSeqNo(data.GetSeqNo());
- w->SetSourceId(NPQ::NSourceIdEncoding::EncodeSimple(SourceId)); // EncodeSimple is needed for compatibility with LB
- //TODO: add in SourceID clientId when TVM will be ready
- w->SetCreateTimeMS(data.GetCreateTimeMs());
- w->SetUncompressedSize(data.GetUncompressedSize());
- };
-
- for (const auto& write : writeRequest->UserWriteRequests) {
- diff -= write->Request.ByteSize();
- if (write->Request.HasDataBatch()) {
- for (const TWriteRequest::TData& data : write->Request.GetDataBatch().GetData()) {
- addData(data);
- }
- } else { // single data
- Y_VERIFY(write->Request.HasData());
- addData(write->Request.GetData());
- }
- }
-
- writeRequest->Cookie = request.GetPartitionRequest().GetCookie();
-
- Y_VERIFY(-diff <= (i64)BytesInflight_);
- diff += request.ByteSize();
- BytesInflight_ += diff;
- BytesInflightTotal_ += diff;
- BytesInflight.Inc(diff);
- BytesInflightTotal.Inc(diff);
-
- writeRequest->ByteSize = request.ByteSize();
- FormedWrites.push_back(writeRequest);
-
- ctx.Send(Writer, std::move(ev));
- ++NumReserveBytesRequests;
-}
-
-TString TWriteSessionActor::CheckSupportedCodec(const ui32 codecId) {
- TString err;
- const auto& description = PQInfo->Description;
- if (!description.GetPQTabletConfig().HasCodecs() || description.GetPQTabletConfig().GetCodecs().IdsSize() == 0)
- return "";
-
- Y_VERIFY(description.PartitionsSize() > 0);
- for (const auto& codec : description.GetPQTabletConfig().GetCodecs().GetIds()) {
- if (codecId == codec) {
- return "";
- }
- }
- err = "Unsupported codec provided. Supported codecs for this topic are:";
- bool first = true;
- for (const auto& codec : description.GetPQTabletConfig().GetCodecs().GetCodecs()) {
- if (first) {
- first = false;
- } else {
- err += ",";
- }
- err += " " + codec;
- }
- return err;
-}
-
-
-void TWriteSessionActor::Handle(TEvPQProxy::TEvWrite::TPtr& ev, const TActorContext& ctx) {
-
- RequestNotChecked = true;
-
- if (State != ES_INITED) {
- //answer error
- CloseSession("write in not inited session", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return;
- }
-
- auto auth = ev->Get()->Request.GetCredentials();
- ev->Get()->Request.ClearCredentials();
- TString tmp;
- Y_PROTOBUF_SUPPRESS_NODISCARD auth.SerializeToString(&tmp);
- if (auth.GetCredentialsCase() != NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET && tmp != AuthStr) {
- Auth = auth;
- AuthStr = tmp;
- ForceACLCheck = true;
- }
- auto dataCheck = [&](const TWriteRequest::TData& data) -> bool {
- if (!data.GetSeqNo()) {
- CloseSession("bad write request - SeqNo must be positive", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return false;
- }
-
- if (data.GetData().empty()) {
- CloseSession("bad write request - data must be non-empty", NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return false;
- }
- TString err = CheckSupportedCodec((ui32)data.GetCodec());
- if (!err.empty()) {
- CloseSession(err, NPersQueue::NErrorCode::BAD_REQUEST, ctx);
- return false;
- }
-
- return true;
- };
- if (ev->Get()->Request.HasDataBatch()) {
- for (const auto& data : ev->Get()->Request.GetDataBatch().GetData()) {
- if (!dataCheck(data)) {
- return;
- }
- }
- } else {
- Y_VERIFY(ev->Get()->Request.HasData());
- if (!dataCheck(ev->Get()->Request.GetData())) {
- return;
- }
- }
-
- THolder<TEvPQProxy::TEvWrite> event(ev->Release());
- Writes.push_back(std::move(event));
-
- ui64 diff = Writes.back()->Request.ByteSize();
- BytesInflight_ += diff;
- BytesInflightTotal_ += diff;
- BytesInflight.Inc(diff);
- BytesInflightTotal.Inc(diff);
-
- if (BytesInflight_ < MAX_BYTES_INFLIGHT) { //allow only one big request to be readed but not sended
- Y_VERIFY(NextRequestInited);
- Handler->ReadyForNextRead();
- } else {
- NextRequestInited = false;
- }
-
- if (NumReserveBytesRequests < MAX_RESERVE_REQUESTS_INFLIGHT) {
- GenerateNextWriteRequest(ctx);
- }
-}
-
-
-void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const TActorContext& ctx) {
- CloseSession(ev->Get()->Reason, ev->Get()->ErrorCode, ctx);
-}
-
-
-void TWriteSessionActor::LogSession(const TActorContext& ctx) {
-
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie
- << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v0 "
- << " topic=" << DiscoveryConverter->GetPrintableString() << " durationSec=" << (ctx.Now() - StartTime).Seconds());
-
- LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60);
-}
-
-void TWriteSessionActor::HandleWakeup(const TActorContext& ctx) {
- Y_VERIFY(State == ES_INITED);
- ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup());
- if (!ACLCheckInProgress && (ForceACLCheck || (ctx.Now() - LastACLCheckTimestamp > TDuration::Seconds(AppData(ctx)->PQConfig.GetACLRetryTimeoutSec()) && RequestNotChecked))) {
- ForceACLCheck = false;
- RequestNotChecked = false;
- if (Auth.GetCredentialsCase() != NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) {
- ACLCheckInProgress = true;
- auto* request = new TEvDescribeTopicsRequest({DiscoveryConverter});
- ctx.Send(SchemeCache, request);
- }
- }
- if (!SourceIdUpdatesInflight && ctx.Now() - LastSourceIdUpdate > SOURCEID_UPDATE_PERIOD) {
- SourceIdUpdatesInflight++;
- Y_VERIFY(FullConverter);
- auto ev = MakeUpdateSourceIdMetadataRequest(EncodedSourceId.Hash, FullConverter->GetPrimaryPath()); // Now Topic is a path
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- // Previously Topic contained legacy name with DC (rt3.dc1--acc--topic)
- SourceIdUpdatesInflight++;
- ev = MakeUpdateSourceIdMetadataRequest(EncodedSourceId.Hash, FullConverter->GetTopicForSrcId());
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- }
- if (ctx.Now() >= LogSessionDeadline) {
- LogSession(ctx);
- }
-}
-
-}
-}
diff --git a/kikimr/yndx/grpc_services/persqueue/persqueue.cpp b/kikimr/yndx/grpc_services/persqueue/persqueue.cpp
deleted file mode 100644
index 43b406a279..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/persqueue.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-#include "persqueue.h"
-#include "grpc_pq_read.h"
-#include "grpc_pq_write.h"
-
-#include <ydb/core/base/appdata.h>
-#include <ydb/core/base/counters.h>
-
-
-namespace NKikimr {
-namespace NGRpcService {
-
-static const ui32 PersQueueWriteSessionsMaxCount = 1000000;
-static const ui32 PersQueueReadSessionsMaxCount = 100000;
-
-TGRpcPersQueueService::TGRpcPersQueueService(NActors::TActorSystem *system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const NActors::TActorId& schemeCache)
- : ActorSystem(system)
- , Counters(counters)
- , SchemeCache(schemeCache)
-{ }
-
-void TGRpcPersQueueService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr) {
- CQ = cq;
- if (ActorSystem->AppData<TAppData>()->PQConfig.GetEnabled()) {
- WriteService.reset(new NGRpcProxy::TPQWriteService(GetService(), CQ, ActorSystem, SchemeCache, Counters, PersQueueWriteSessionsMaxCount));
- WriteService->InitClustersUpdater();
- ReadService.reset(new NGRpcProxy::TPQReadService(this, CQ, ActorSystem, SchemeCache, Counters, PersQueueReadSessionsMaxCount));
- SetupIncomingRequests();
- }
-}
-
-void TGRpcPersQueueService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) {
- Limiter = limiter;
-}
-
-bool TGRpcPersQueueService::IncRequest() {
- return Limiter->Inc();
-}
-
-void TGRpcPersQueueService::DecRequest() {
- Limiter->Dec();
-}
-
-void TGRpcPersQueueService::SetupIncomingRequests() {
- WriteService->SetupIncomingRequests();
- ReadService->SetupIncomingRequests();
-}
-
-void TGRpcPersQueueService::StopService() noexcept {
- TGrpcServiceBase::StopService();
- if (WriteService.get() != nullptr) {
- WriteService->StopService();
- }
- if (ReadService.get() != nullptr) {
- ReadService->StopService();
- }
-}
-
-} // namespace NGRpcService
-} // namespace NKikimr
diff --git a/kikimr/yndx/grpc_services/persqueue/persqueue.h b/kikimr/yndx/grpc_services/persqueue/persqueue.h
deleted file mode 100644
index 267efa7a6d..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/persqueue.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#pragma once
-
-#include <library/cpp/actors/core/actorsystem.h>
-
-#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h>
-
-#include <library/cpp/grpc/server/grpc_server.h>
-
-
-namespace NKikimr {
-
-namespace NGRpcProxy {
- class TPQWriteService;
- class TPQReadService;
-}
-
-namespace NGRpcService {
-
-class TGRpcPersQueueService
- : public NGrpc::TGrpcServiceBase<NPersQueue::PersQueueService>
-{
-public:
- TGRpcPersQueueService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const NActors::TActorId& schemeCache);
-
- void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override;
- void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override;
- void StopService() noexcept override;
-
- using NGrpc::TGrpcServiceBase<NPersQueue::PersQueueService>::GetService;
-
- bool IncRequest();
- void DecRequest();
-
-private:
- void SetupIncomingRequests();
-
- NActors::TActorSystem* ActorSystem;
- grpc::ServerCompletionQueue* CQ = nullptr;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- NGrpc::TGlobalLimiter* Limiter = nullptr;
- NActors::TActorId SchemeCache;
-
- std::shared_ptr<NGRpcProxy::TPQWriteService> WriteService;
- std::shared_ptr<NGRpcProxy::TPQReadService> ReadService;
-};
-
-} // namespace NGRpcService
-} // namespace NKikimr
diff --git a/kikimr/yndx/grpc_services/persqueue/persqueue_compat_ut.cpp b/kikimr/yndx/grpc_services/persqueue/persqueue_compat_ut.cpp
deleted file mode 100644
index 7b9b117bcd..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/persqueue_compat_ut.cpp
+++ /dev/null
@@ -1,122 +0,0 @@
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h>
-
-namespace NKikimr::NPersQueueTests {
-
-using namespace NPersQueue::NTests;
-
-class TPQv0CompatTestBase {
-public:
- THolder<TTestPQLib> PQLib;
- THolder<::NPersQueue::TTestServer> Server;
- TString OriginalLegacyName1;
- TString OriginalModernName1;
- TString MirroredLegacyName1;
- TString MirroredModernName1;
- TString ShortName1;
-
- TString OriginalLegacyName2;
- TString OriginalModernName2;
- TString MirroredLegacyName2;
- TString MirroredModernName2;
- TString ShortName2;
-
-public:
- TPQv0CompatTestBase()
- {
- Server = MakeHolder<::NPersQueue::TTestServer>(false);
- Server->ServerSettings.PQConfig.MutablePQDiscoveryConfig()->SetLbUserDatabaseRoot("/Root/LB");
- Server->ServerSettings.PQConfig.SetCheckACL(false);
- Server->StartServer();
- Server->EnableLogs({ NKikimrServices::KQP_PROXY }, NActors::NLog::PRI_EMERG);
- Server->EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
- Server->EnableLogs({ NKikimrServices::PQ_METACACHE }, NActors::NLog::PRI_DEBUG);
- OriginalLegacyName1 = "rt3.dc1--account--topic1";
- MirroredLegacyName1 = "rt3.dc2--account--topic1";
- OriginalModernName1 = "/Root/LB/account/topic1";
- MirroredModernName1 = "/Root/LB/account/.topic2/mirrored-from-dc2";
- ShortName1 = "account/topic1";
-
- OriginalLegacyName2 = "rt3.dc1--account--topic2";
- MirroredLegacyName2 = "rt3.dc2--account--topic2";
- OriginalModernName2 = "/Root/LB/account/topic2";
- MirroredModernName2 = "/Root/LB/account/.topic2/mirrored-from-dc2";
- ShortName2 = "account/topic2";
-
- Server->AnnoyingClient->CreateTopicNoLegacy(OriginalLegacyName1, 1, false);
- Server->AnnoyingClient->CreateTopicNoLegacy(MirroredLegacyName1, 1, false);
- Server->AnnoyingClient->CreateTopicNoLegacy(OriginalModernName2, 1, true, true, "dc1");
- Server->AnnoyingClient->CreateTopicNoLegacy(MirroredModernName2, 1, true, true, "dc2");
- Server->AnnoyingClient->CreateConsumer("test-consumer");
- InitPQLib();
- }
- void InitPQLib() {
- PQLib = MakeHolder<TTestPQLib>(*Server);
- TPQDataWriter writer{OriginalLegacyName1, ShortName1, "test", *Server};
- writer.WaitWritePQServiceInitialization();
- };
-};
-
-Y_UNIT_TEST_SUITE(TPQCompatTest) {
- Y_UNIT_TEST(DiscoverTopics) {
- TPQv0CompatTestBase testServer;
- Cerr << "Create producer\n";
- {
- auto [producer, res] = testServer.PQLib->CreateProducer(testServer.ShortName2, "123", {}, ::NPersQueue::ECodec::RAW);
- Cerr << "Got response: " << res.Response.ShortDebugString() << Endl;
- UNIT_ASSERT(res.Response.HasInit());
- }
- Cerr << "Create producer(2)\n";
- {
- auto [producer, res] = testServer.PQLib->CreateProducer(testServer.ShortName1, "123", {}, ::NPersQueue::ECodec::RAW);
- UNIT_ASSERT(res.Response.HasInit());
- }
- }
-
- Y_UNIT_TEST(SetupLockSession) {
- TPQv0CompatTestBase server{};
- auto [consumer, startResult] = server.PQLib->CreateConsumer({server.ShortName1}, "test-consumer", 1, true);
- Cerr << startResult.Response << "\n";
- for (ui32 i = 0; i < 2; ++i) {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << "Response: " << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == server.OriginalLegacyName1
- || msg.GetValue().Response.GetLock().GetTopic() == server.MirroredLegacyName1);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- }
- auto msg = consumer->GetNextMessage();
- UNIT_ASSERT(!msg.Wait(TDuration::Seconds(1)));
- server.Server->AnnoyingClient->AlterTopic(server.MirroredLegacyName1, 2);
- msg.Wait();
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == server.MirroredLegacyName1);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 1);
- }
-
- Y_UNIT_TEST(LegacyRequests) {
- TPQv0CompatTestBase server{};
- server.Server->AnnoyingClient->GetPartOffset(
- {
- {server.OriginalLegacyName1, {0}},
- {server.MirroredLegacyName1, {0}},
- {server.OriginalLegacyName2, {0}},
- {server.MirroredLegacyName2, {0}},
- },
- 4, 0, true
- );
- server.Server->AnnoyingClient->SetClientOffsetPQ(server.OriginalLegacyName2, 0, 5);
- server.Server->AnnoyingClient->SetClientOffsetPQ(server.MirroredLegacyName2, 0, 5);
-
- server.Server->AnnoyingClient->GetPartOffset(
- {
- {server.OriginalLegacyName2, {0}},
- {server.MirroredLegacyName2, {0}},
- },
- 2, 2, true
- );
- }
-}
-} //namespace NKikimr::NPersQueueTests;
diff --git a/kikimr/yndx/grpc_services/persqueue/persqueue_ut.cpp b/kikimr/yndx/grpc_services/persqueue/persqueue_ut.cpp
deleted file mode 100644
index 6a899a684d..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/persqueue_ut.cpp
+++ /dev/null
@@ -1,2405 +0,0 @@
-#include "ut/definitions.h"
-#include <ydb/core/base/appdata.h>
-#include <ydb/core/testlib/test_pq_client.h>
-#include <ydb/core/client/server/grpc_proxy_status.h>
-#include <ydb/core/protos/grpc_pq_old.pb.h>
-#include <ydb/core/persqueue/writer/source_id_encoding.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-
-#include <ydb/library/aclib/aclib.h>
-#include <ydb/library/persqueue/tests/counters.h>
-
-#include <library/cpp/testing/unittest/tests_data.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/http/io/stream.h>
-#include <google/protobuf/text_format.h>
-
-#include <util/string/join.h>
-#include <util/string/builder.h>
-
-#include <grpc++/client_context.h>
-#include <grpc++/create_channel.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-
-namespace NKikimr {
-namespace NPersQueueTests {
-
-using namespace Tests;
-using namespace NKikimrClient;
-using namespace NPersQueue;
-using namespace NPersQueue::NTests;
-using namespace NThreading;
-using namespace NNetClassifier;
-
-static TString FormNetData() {
- return "10.99.99.224/32\tSAS\n"
- "::1/128\tVLA\n";
-}
-
-TAutoPtr<IEventHandle> GetClassifierUpdate(TServer& server, const TActorId sender) {
- auto& actorSystem = *server.GetRuntime();
- actorSystem.Send(
- new IEventHandle(MakeNetClassifierID(), sender,
- new TEvNetClassifier::TEvSubscribe()
- ));
-
- TAutoPtr<IEventHandle> handle;
- actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(handle);
-
- UNIT_ASSERT(handle);
- UNIT_ASSERT_VALUES_EQUAL(handle->Recipient, sender);
-
- return handle;
-}
-
-THolder<TTempFileHandle> CreateNetDataFile(const TString& content) {
- auto netDataFile = MakeHolder<TTempFileHandle>("data.tsv");
-
- netDataFile->Write(content.Data(), content.Size());
- netDataFile->FlushData();
-
- return netDataFile;
-}
-
-
-Y_UNIT_TEST_SUITE(TPersQueueTest2) {
- void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) {
- annoyingClient.SetNoConfigMode();
- annoyingClient.FullInit();
- annoyingClient.InitUserRegistry();
- annoyingClient.MkDir("/Root", "account1");
- annoyingClient.MkDir("/Root/PQ", "account1");
- annoyingClient.CreateTopicNoLegacy("/Root/PQ/rt3.db--topic1", 5, false);
- annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false, true, Nothing(), {"user1", "user2"});
- annoyingClient.CreateTopicNoLegacy("/Root/account2/topic2", 5);
- }
- Y_UNIT_TEST(TestGrpcWriteNoDC) {
- TTestServer server(false);
- server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- server.ServerSettings.PQConfig.SetRoot("/Rt2/PQ");
- server.ServerSettings.PQConfig.SetDatabase("/Root");
- server.StartServer();
-
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"folder_id", "somefolder"}, {"cloud_id", "somecloud"}, {"database_id", "root"}}));
-
-
- PrepareForGrpcNoDC(*server.AnnoyingClient);
- auto writer = MakeDataWriter(server, "source1");
-
- writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
-
- NACLib::TDiffACL acl;
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN);
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "topic1@" BUILTIN_ACL_DOMAIN);
-
- server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString());
- server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString());
-
- Sleep(TDuration::Seconds(5));
-
- writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
-
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
-
- writer.Read("/Root/PQ/account1/topic1", "user1", "topic1@" BUILTIN_ACL_DOMAIN, false, false, false, true);
-
- writer.Write("Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN); //TODO /Root remove
-
- writer.Read("Root/PQ/account1/topic1", "user1", "topic1@" BUILTIN_ACL_DOMAIN, false, false, false, true);
- }
-}
-Y_UNIT_TEST_SUITE(TPersQueueTest) {
-
- Y_UNIT_TEST(SetupLockSession2) {
- TTestServer server(false);
- server.GrpcServerOptions.SetMaxMessageSize(130_MB);
- server.StartServer();
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
- server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1);
- server.AnnoyingClient->CreateTopic("rt3.dc2--acc--topic1", 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.AnnoyingClient->CreateConsumer("user");
-
- auto writer = MakeDataWriter(server);
-
- TTestPQLib PQLib(server);
- auto [consumer, startResult] = PQLib.CreateConsumer({"acc/topic1"}, "user", 1, true);
- Cerr << startResult.Response << "\n";
- for (ui32 i = 0; i < 2; ++i) {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << "Response: " << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == "rt3.dc1--acc--topic1" || msg.GetValue().Response.GetLock().GetTopic() == "rt3.dc2--acc--topic1");
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- }
- auto msg = consumer->GetNextMessage();
- UNIT_ASSERT(!msg.Wait(TDuration::Seconds(1)));
- server.AnnoyingClient->AlterTopic("rt3.dc2--acc--topic1", 2);
- msg.Wait();
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == "rt3.dc2--acc--topic1");
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 1);
- }
-
-
-
- Y_UNIT_TEST(SetupLockSession) {
- TTestServer server;
-
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
-
- server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.AnnoyingClient->CreateConsumer("user");
-
- auto writer = MakeDataWriter(server);
-
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_;
- std::unique_ptr<NPersQueue::PersQueueService::Stub> StubP_;
-
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_);
-
- ui64 proxyCookie = 0;
-
- {
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- NKikimrClient::TResponse response;
- auto status = Stub_->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- proxyCookie = response.GetProxyCookie();
- Channel_ = grpc::CreateChannel("[" + response.GetProxyName() + "]:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- StubP_ = NPersQueue::PersQueueService::NewStub(Channel_);
- }
-
-
- grpc::ClientContext rcontext;
- auto readStream = StubP_->ReadSession(&rcontext);
- UNIT_ASSERT(readStream);
-
- // init read session
- {
- TReadRequest req;
- TReadResponse resp;
-
- req.MutableInit()->AddTopics("acc/topic1");
-
- req.MutableInit()->SetClientId("user");
- req.MutableInit()->SetClientsideLocksAllowed(true);
- req.MutableInit()->SetProxyCookie(proxyCookie);
- req.MutableInit()->SetProtocolVersion(TReadRequest::ReadParamsInInit);
- req.MutableInit()->SetMaxReadMessagesCount(3);
-
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasInit());
- //send some reads
- req.Clear();
- req.MutableRead();
- for (ui32 i = 0; i < 10; ++i) {
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- }
- }
-
- {
- TReadRequest req;
- TReadResponse resp;
-
- //lock partition
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasLock());
- UNIT_ASSERT_VALUES_EQUAL(resp.GetLock().GetTopic(), "rt3.dc1--acc--topic1");
- UNIT_ASSERT(resp.GetLock().GetPartition() == 0);
-
- req.Clear();
- req.MutableStartRead()->SetTopic(resp.GetLock().GetTopic());
- req.MutableStartRead()->SetPartition(resp.GetLock().GetPartition());
- req.MutableStartRead()->SetReadOffset(10);
- req.MutableStartRead()->SetGeneration(resp.GetLock().GetGeneration());
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
-
- }
-
- //Write some data
- writer.Write("acc/topic1", "valuevaluevalue1");
- Sleep(TDuration::Seconds(15)); //force wait data
- writer.Write("acc/topic1", "valuevaluevalue2");
- writer.Write("acc/topic1", "valuevaluevalue3");
- writer.Write("acc/topic1", "valuevaluevalue4");
-
- //check read results
- TReadResponse resp;
- for (ui32 i = 10; i < 16; ++i) {
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT_C(resp.HasBatchedData(), resp);
- UNIT_ASSERT(resp.GetBatchedData().PartitionDataSize() == 1);
- UNIT_ASSERT(resp.GetBatchedData().GetPartitionData(0).BatchSize() == 1);
- UNIT_ASSERT(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).MessageDataSize() == 1);
- UNIT_ASSERT(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).GetMessageData(0).GetOffset() == i);
- }
- //TODO: restart here readSession and read from position 10
- {
- TReadRequest req;
- TReadResponse resp;
-
- req.MutableCommit()->AddCookie(1);
-
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasCommit());
- }
- }
-
-
- void SetupWriteSessionImpl(bool rr) {
- TTestServer server(PQSettings(0, 2, rr));
-
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- auto writer = MakeDataWriter(server);
-
- ui32 p = writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue1");
-
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 15);
-
- ui32 pp = writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue2");
- UNIT_ASSERT_VALUES_EQUAL(p, pp);
-
- writer.WriteBatch(SHORT_TOPIC_NAME, {"1", "2", "3", "4", "5"});
-
- writer.Write("topic2", "valuevaluevalue1", true);
-
- p = writer.InitSession("sid1", 2, true);
- pp = writer.InitSession("sid1", 0, true);
-
- UNIT_ASSERT(p = pp);
- UNIT_ASSERT(p == 1);
-
- {
- p = writer.InitSession("sidx", 0, true);
- pp = writer.InitSession("sidx", 0, true);
-
- UNIT_ASSERT(p == pp);
- }
-
- writer.InitSession("sid1", 3, false);
-
- //check round robin;
- TMap<ui32, ui32> ss;
- for (ui32 i = 0; i < 15*5; ++i) {
- ss[writer.InitSession("sid_rand_" + ToString<ui32>(i), 0, true)]++;
- }
- for (auto &s : ss) {
- Cerr << s.first << " " << s.second << "\n";
- if (rr) {
- UNIT_ASSERT(s.second >= 4 && s.second <= 6);
- }
- }
- }
-
- Y_UNIT_TEST(SetupWriteSession) {
- SetupWriteSessionImpl(false);
- SetupWriteSessionImpl(true);
- }
-
- Y_UNIT_TEST(SetupWriteSessionOnDisabledCluster) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_WRITE_PROXY});
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
-
- auto writer = MakeDataWriter(server);
-
- server.AnnoyingClient->DisableDC();
-
- Sleep(TDuration::Seconds(5));
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue1", true);
- }
-
- Y_UNIT_TEST(CloseActiveWriteSessionOnClusterDisable) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- auto writer = MakeDataWriter(server);
-
- TTestPQLib PQLib(server);
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123", {}, ECodec::RAW);
-
- NThreading::TFuture<NPersQueue::TError> isDead = producer->IsDead();
- server.AnnoyingClient->DisableDC();
- isDead.Wait();
- UNIT_ASSERT_EQUAL(isDead.GetValue().GetCode(), NPersQueue::NErrorCode::CLUSTER_DISABLED);
- }
-
- Y_UNIT_TEST(BadSids) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
- auto runSidTest = [&](const TString& srcId, bool shouldFail = true) {
- auto[producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, srcId);
- if (shouldFail) {
- UNIT_ASSERT(res.Response.HasError());
- } else {
- UNIT_ASSERT(res.Response.HasInit());
- }
- };
-
- runSidTest("base64:a***");
- runSidTest("base64:aa==");
- runSidTest("base64:a");
- runSidTest("base64:aa", false);
- }
-
- Y_UNIT_TEST(ReadFromSeveralPartitions) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- auto writer = MakeDataWriter(server, "source1");
-
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_;
- std::unique_ptr<NPersQueue::PersQueueService::Stub> StubP_;
-
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_);
-
- ui64 proxyCookie = 0;
-
- {
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- NKikimrClient::TResponse response;
- auto status = Stub_->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- proxyCookie = response.GetProxyCookie();
- Channel_ = grpc::CreateChannel("[" + response.GetProxyName() + "]:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- StubP_ = NPersQueue::PersQueueService::NewStub(Channel_);
- }
-
-
- //Write some data
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue1");
-
- auto writer2 = MakeDataWriter(server, "source2");
- writer2.Write(SHORT_TOPIC_NAME, "valuevaluevalue2");
-
- grpc::ClientContext rcontext;
- auto readStream = StubP_->ReadSession(&rcontext);
- UNIT_ASSERT(readStream);
-
- // init read session
- {
- TReadRequest req;
- TReadResponse resp;
-
- req.MutableInit()->AddTopics(SHORT_TOPIC_NAME);
-
- req.MutableInit()->SetClientId("user");
- req.MutableInit()->SetProxyCookie(proxyCookie);
- req.MutableInit()->SetProtocolVersion(TReadRequest::ReadParamsInInit);
- req.MutableInit()->SetMaxReadMessagesCount(1000);
-
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasInit());
-
- //send some reads
- Sleep(TDuration::Seconds(5));
- for (ui32 i = 0; i < 10; ++i) {
- req.Clear();
- req.MutableRead();
-
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- }
- }
-
- //check read results
- TReadResponse resp;
- for (ui32 i = 0; i < 1; ++i) {
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT_C(resp.HasBatchedData(), resp);
- UNIT_ASSERT(resp.GetBatchedData().PartitionDataSize() == 2);
- }
- }
-
-
- void SetupReadSessionTest(bool useBatching) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->CreateTopic("rt3.dc2--topic1", 2);
-
- auto writer = MakeDataWriter(server, "source1");
-
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue0");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue1");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue2");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue3");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue4");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue5");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue6");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue7");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue8");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue9");
-
- writer.Read(SHORT_TOPIC_NAME, "user", "", false, false, useBatching);
- }
-
- Y_UNIT_TEST(SetupReadSession) {
- SetupReadSessionTest(false);
- }
-
- Y_UNIT_TEST(SetupReadSessionWithBatching) {
- SetupReadSessionTest(true);
- }
-
- void ClosesSessionOnReadSettingsChangeTest(bool initReadSettingsInInitRequest) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->CreateTopic("rt3.dc2--topic1", 2);
-
- auto writer = MakeDataWriter(server, "source1");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue0");
-
- // Reading code
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_;
- std::unique_ptr<NPersQueue::PersQueueService::Stub> StubP_;
-
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_);
-
- ui64 proxyCookie = 0;
-
- {
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- NKikimrClient::TResponse response;
- auto status = Stub_->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- proxyCookie = response.GetProxyCookie();
- Channel_ = grpc::CreateChannel("[" + response.GetProxyName() + "]:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- StubP_ = NPersQueue::PersQueueService::NewStub(Channel_);
- }
-
- grpc::ClientContext rcontext;
- auto readStream = StubP_->ReadSession(&rcontext);
- UNIT_ASSERT(readStream);
-
- // init read session
- {
- TReadRequest req;
- TReadResponse resp;
-
- req.MutableInit()->AddTopics(SHORT_TOPIC_NAME);
-
- req.MutableInit()->SetClientId("user");
- req.MutableInit()->SetProxyCookie(proxyCookie);
- if (initReadSettingsInInitRequest) {
- req.MutableInit()->SetProtocolVersion(TReadRequest::ReadParamsInInit);
- req.MutableInit()->SetMaxReadMessagesCount(1);
- }
-
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasInit());
-
- if (!initReadSettingsInInitRequest) {
- // send first read
- req.Clear();
- req.MutableRead()->SetMaxCount(1);
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT_C(resp.HasData(), resp);
- }
-
- // change settings
- req.Clear();
- req.MutableRead()->SetMaxCount(42);
- if (!readStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- UNIT_ASSERT(readStream->Read(&resp));
- UNIT_ASSERT(resp.HasError());
- }
- }
-
- Y_UNIT_TEST(WriteSessionClose) {
-
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->CreateTopic("rt3.dc2--topic1", 2);
-
- auto writer = MakeDataWriter(server, "source1");
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue0");
-
- // Reading code
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_;
- std::unique_ptr<NPersQueue::PersQueueService::Stub> StubP_;
-
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_);
-
- ui64 proxyCookie = 0;
-
- {
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- NKikimrClient::TResponse response;
- auto status = Stub_->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- proxyCookie = response.GetProxyCookie();
- Channel_ = grpc::CreateChannel("[" + response.GetProxyName() + "]:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- StubP_ = NPersQueue::PersQueueService::NewStub(Channel_);
- }
-
- // init write session
- for (ui32 i = 0; i < 2; ++i){
- grpc::ClientContext rcontext;
-
- auto writeStream = StubP_->WriteSession(&rcontext);
- UNIT_ASSERT(writeStream);
-
- TWriteRequest req;
-
- req.MutableInit()->SetTopic(SHORT_TOPIC_NAME);
-
- req.MutableInit()->SetSourceId("user");
- req.MutableInit()->SetProxyCookie(proxyCookie);
- if (i == 0)
- continue;
- if (!writeStream->Write(req)) {
- ythrow yexception() << "write fail";
- }
- }
- }
-
- Y_UNIT_TEST(ClosesSessionOnReadSettingsChange) {
- ClosesSessionOnReadSettingsChangeTest(false);
- }
-
- Y_UNIT_TEST(ClosesSessionOnReadSettingsChangeWithInit) {
- ClosesSessionOnReadSettingsChangeTest(true);
- }
-
- Y_UNIT_TEST(WriteExisting) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- {
- THolder<NMsgBusProxy::TBusPersQueue> request = TRequestDescribePQ().GetRequest({});
-
- NKikimrClient::TResponse response;
-
- auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- auto stub(NKikimrClient::TGRpcServer::NewStub(channel));
- grpc::ClientContext context;
- auto status = stub->PersQueueRequest(&context, request->Record, &response);
-
- UNIT_ASSERT(status.ok());
- }
-
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::GRpc
- );
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "", ETransport::GRpc);
- }
-
- Y_UNIT_TEST(WriteExistingBigValue) {
- TTestServer server(PQSettings(0, 2).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8_MB, 86400, 100000);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- TInstant now(Now());
-
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, TString(1000000, 'a'));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, TString(1, 'a'));
- UNIT_ASSERT(TInstant::Now() - now > TDuration::MilliSeconds(5990)); //speed limit is 200kb/s and burst is 200kb, so to write 1mb it will take at least 4 seconds
- }
-
- Y_UNIT_TEST(WriteEmptyData) {
- TTestServer server(PQSettings(0, 2).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- // empty data and sourecId
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_OK);
- }
-
-
- Y_UNIT_TEST(WriteNonExistingPartition) {
- TTestServer server(PQSettings(0, 2).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 100500, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
- }
-
- Y_UNIT_TEST(WriteNonExistingTopic) {
- TTestServer server(PQSettings(0, 2).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- server.AnnoyingClient->WriteToPQ("rt3.dc1--topic1000", 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR);
- }
-
- Y_UNIT_TEST(SchemeshardRestart) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- TString topic2 = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(topic2, 2);
-
- // force topic1 into cache and establish pipe from cache to schemeshard
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
- server.AnnoyingClient->RestartSchemeshard(server.CleverServer->GetRuntime());
- server.AnnoyingClient->WriteToPQ(topic2, 1, "abacaba", 1, "valuevaluevalue1");
- }
-
- Y_UNIT_TEST(WriteAfterAlter) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
-
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
-
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 10);
- Sleep(TDuration::Seconds(1));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
-
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
- Sleep(TDuration::Seconds(1));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1");
- }
-
- Y_UNIT_TEST(Delete) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- // Delete non-existing
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
-
- // Delete existing
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
-
- // Double delete - "What Is Dead May Never Die"
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
-
- // Resurrect deleted topic
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
- }
-
- Y_UNIT_TEST(WriteAfterDelete) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
-
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 2, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
- }
-
- Y_UNIT_TEST(WriteAfterCreateDeleteCreate) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- TString topic = "rt3.dc1--johnsnow";
- server.AnnoyingClient->CreateTopic(topic, 2);
-
- server.AnnoyingClient->WriteToPQ(topic, 1, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(topic, 3, "abacaba", 1, "valuevaluevalue1", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
- server.AnnoyingClient->DeleteTopic2(topic);
-
- server.AnnoyingClient->CreateTopic(topic, 4);
-
- // Write to topic, cache must be updated by CreateTopic
- server.AnnoyingClient->WriteToPQ(topic, 1, "abacaba", 1, "valuevaluevalue1");
- // Write to partition that didn't exist in the old topic
- server.AnnoyingClient->WriteToPQ(topic, 3, "abacaba", 1, "valuevaluevalue1");
- }
-
- Y_UNIT_TEST(GetOffsetsAfterDelete) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- TString topic2 = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- server.AnnoyingClient->CreateTopic(topic2, 3);
-
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
-
- // Get offsets from deleted topic
- server.AnnoyingClient->GetPartOffset( {
- {DEFAULT_TOPIC_NAME, {1,2}}
- }, 0, 0, false);
-
- // Get offsets from multiple topics
- server.AnnoyingClient->GetPartOffset( {
- {DEFAULT_TOPIC_NAME, {1,2}},
- {topic2, {1,2}},
- }, 0, 0, false);
- }
-
-
- Y_UNIT_TEST(GetOffsetsAfterCreateDeleteCreate) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- TString topic2 = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- server.AnnoyingClient->CreateTopic(topic2, 3);
-
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE });
-
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
- Sleep(TDuration::Seconds(1));
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 4);
- Sleep(TDuration::Seconds(1));
-
- // Get offsets from multiple topics
- server.AnnoyingClient->GetPartOffset( {
- {DEFAULT_TOPIC_NAME, {1,2}},
- {topic2, {1}},
- }, 3, 0, true);
- }
-
- Y_UNIT_TEST(BigRead) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8_MB, 86400, 20000000, "user", 2000000);
-
- server.EnableLogs( { NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- TString value(1_MB, 'x');
- for (ui32 i = 0; i < 32; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
-
- // trying to read small PQ messages in a big messagebus event
- auto info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
- UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromDisk, 0);
- UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromCache, 4);
-
- TInstant now(TInstant::Now());
- info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
- TDuration dur = TInstant::Now() - now;
- UNIT_ASSERT_C(dur > TDuration::Seconds(7) && dur < TDuration::Seconds(20), "dur = " << dur); //speed limit is 2000kb/s and burst is 2000kb, so to read 24mb it will take at least 11 seconds
-
- server.AnnoyingClient->GetPartStatus({}, 1, true);
-
- }
-
- // expects that L2 size is 32Mb
- Y_UNIT_TEST(Cache) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8_MB);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- TString value(1_MB, 'x');
- for (ui32 i = 0; i < 32; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
-
- auto info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user"}, 16);
- auto info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user"}, 16);
-
- UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromCache, 3);
- UNIT_ASSERT_VALUES_EQUAL(info16.BlobsFromCache, 2);
- UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromDisk + info16.BlobsFromDisk, 0);
-
- for (ui32 i = 0; i < 8; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 32+i}, value);
-
- info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user"}, 16);
- info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user"}, 16);
-
- ui32 fromDisk = info0.BlobsFromDisk + info16.BlobsFromDisk;
- ui32 fromCache = info0.BlobsFromCache + info16.BlobsFromCache;
- UNIT_ASSERT(fromDisk > 0);
- UNIT_ASSERT(fromDisk < 5);
- UNIT_ASSERT(fromCache > 0);
- UNIT_ASSERT(fromCache < 5);
- }
-
- Y_UNIT_TEST(CacheHead) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6_MB);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- ui64 seqNo = 0;
- for (ui32 blobSizeKB = 256; blobSizeKB < 4096; blobSizeKB *= 2) {
- static const ui32 maxEventKB = 24_KB;
- ui32 blobSize = blobSizeKB * 1_KB;
- ui32 count = maxEventKB / blobSizeKB;
- count -= count%2;
- ui32 half = count/2;
-
- ui64 offset = seqNo;
- TString value(blobSize, 'a');
- for (ui32 i = 0; i < count; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", seqNo++}, value);
-
- auto info_half1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
- auto info_half2 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
-
- UNIT_ASSERT(info_half1.BlobsFromCache > 0);
- UNIT_ASSERT(info_half2.BlobsFromCache > 0);
- UNIT_ASSERT_VALUES_EQUAL(info_half1.BlobsFromDisk, 0);
- UNIT_ASSERT_VALUES_EQUAL(info_half2.BlobsFromDisk, 0);
- }
- }
-
- Y_UNIT_TEST(SameOffset) {
- TTestServer server;
- TString topic2 = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6_MB);
- server.AnnoyingClient->CreateTopic(topic2, 1, 6_MB);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
-
- ui32 valueSize = 128;
- TString value1(valueSize, 'a');
- TString value2(valueSize, 'b');
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 0}, value1);
- server.AnnoyingClient->WriteToPQ({topic2, 0, "source1", 0}, value2);
-
- // avoid reading from head
- TString mb(1_MB, 'x');
- for (ui32 i = 1; i < 16; ++i) {
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, mb);
- server.AnnoyingClient->WriteToPQ({topic2, 0, "source1", i}, mb);
- }
-
- auto info1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 1, "user1"}, 1);
- auto info2 = server.AnnoyingClient->ReadFromPQ({topic2, 0, 0, 1, "user1"}, 1);
-
- UNIT_ASSERT_VALUES_EQUAL(info1.BlobsFromCache, 1);
- UNIT_ASSERT_VALUES_EQUAL(info2.BlobsFromCache, 1);
- UNIT_ASSERT_VALUES_EQUAL(info1.Values.size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(info2.Values.size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(info1.Values[0].size(), valueSize);
- UNIT_ASSERT_VALUES_EQUAL(info2.Values[0].size(), valueSize);
- UNIT_ASSERT(info1.Values[0] == value1);
- UNIT_ASSERT(info2.Values[0] == value2);
- }
-
-
- Y_UNIT_TEST(FetchRequest) {
- TTestServer server;
- TString topic2 = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.AnnoyingClient->CreateTopic(topic2, 10);
-
- ui32 valueSize = 128;
- TString value1(valueSize, 'a');
- TString value2(valueSize, 'b');
- server.AnnoyingClient->WriteToPQ({topic2, 5, "source1", 0}, value2);
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 0}, value1);
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 1}, value2);
-
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD });
- TInstant tm(TInstant::Now());
- server.AnnoyingClient->FetchRequestPQ(
- { {topic2, 5, 0, 400}, {DEFAULT_TOPIC_NAME, 1, 0, 400}, {DEFAULT_TOPIC_NAME, 3, 0, 400} },
- 400, 1000000
- );
- UNIT_ASSERT((TInstant::Now() - tm).Seconds() < 1);
- tm = TInstant::Now();
- server.AnnoyingClient->FetchRequestPQ({{topic2, 5, 1, 400}}, 400, 5000);
- UNIT_ASSERT((TInstant::Now() - tm).Seconds() > 2);
- server.AnnoyingClient->FetchRequestPQ(
- { {topic2, 5, 0, 400}, {DEFAULT_TOPIC_NAME, 1, 0, 400}, {DEFAULT_TOPIC_NAME, 3, 0, 400} },
- 1, 1000000
- );
- server.AnnoyingClient->FetchRequestPQ(
- { {topic2, 5, 500, 400}, {topic2, 4, 0, 400}, {DEFAULT_TOPIC_NAME, 1, 0, 400} },
- 400, 1000000
- );
- }
-
- Y_UNIT_TEST(ChooseProxy) {
- TTestServer server;
- server.AnnoyingClient->ChooseProxy(ETransport::GRpc);
- }
-
-
- Y_UNIT_TEST(Init) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- TString topic2 = "rt3.dc1--topic2";
- TString topic3 = "rt3.dc1--topic3";
-
- if (!true) {
- server.EnableLogs({
- NKikimrServices::FLAT_TX_SCHEMESHARD,
- NKikimrServices::TX_DATASHARD,
- NKikimrServices::HIVE,
- NKikimrServices::PERSQUEUE,
- NKikimrServices::TABLET_MAIN,
- NKikimrServices::BS_PROXY_DISCOVER,
- NKikimrServices::PIPE_CLIENT,
- NKikimrServices::PQ_METACACHE });
- }
-
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->TestCase({}, 0, 0, true);
-
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
- server.AnnoyingClient->CreateTopic(topic2, 25);
-
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 2, "valuevaluevalue2");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacabae", 1, "valuevaluevalue3");
- server.AnnoyingClient->ReadFromPQ(DEFAULT_TOPIC_NAME, 5, 0, 10, 3);
-
- server.AnnoyingClient->SetClientOffsetPQ(DEFAULT_TOPIC_NAME, 5, 2);
-
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5}}}, 1, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0}}}, 1, 0, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}}, 20, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5, 5}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {111}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({}, 45, 1, true);
- server.AnnoyingClient->TestCase({{topic3, {}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {topic3, {}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {topic2, {}}}, 45, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0, 3, 5}}, {topic2, {1, 4, 6, 8}}}, 7, 1, true);
-
- server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME});
- server.AnnoyingClient->DescribeTopic({topic2});
- server.AnnoyingClient->DescribeTopic({topic2, DEFAULT_TOPIC_NAME});
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->DescribeTopic({topic3}, true);
- }
-
-
- Y_UNIT_TEST(DescribeAfterDelete) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::PQ_METACACHE });
- TString topic2 = "rt3.dc1--topic2";
-
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.AnnoyingClient->CreateTopic(topic2, 10);
- server.AnnoyingClient->DescribeTopic({});
-
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
-
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->GetClientInfo({}, "user", true);
- server.AnnoyingClient->GetClientInfo({topic2}, "user", true);
- Sleep(TDuration::Seconds(2));
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {topic2, {}}}, 10, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {topic2, {}}, {"rt3.dc1--topic3", {}}}, 10, 0, false);
- }
-
- Y_UNIT_TEST(DescribeAfterDelete2) {
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::PQ_METACACHE });
- TString topic3 = "rt3.dc1--topic3";
-
- server.AnnoyingClient->CreateTopic(topic3, 10);
- Sleep(TDuration::Seconds(3)); //for invalidation of cache
- server.AnnoyingClient->TestCase({{"rt3.dc1--topic4", {}}}, 10, 0, false); //will force caching of topic3
- server.AnnoyingClient->DeleteTopic2(topic3);
- server.AnnoyingClient->DescribeTopic({topic3}, true); //describe will fail
- server.AnnoyingClient->DescribeTopic({topic3}, true); //describe will fail
- }
-
-
- void WaitResolveSuccess(TTestServer& server, TString topic, ui32 numParts) {
- const TInstant start = TInstant::Now();
- while (true) {
- TAutoPtr<NMsgBusProxy::TBusPersQueue> request(new NMsgBusProxy::TBusPersQueue);
- auto req = request->Record.MutableMetaRequest();
- auto partOff = req->MutableCmdGetPartitionLocations();
- auto treq = partOff->AddTopicRequest();
- treq->SetTopic(topic);
- for (ui32 i = 0; i < numParts; ++i)
- treq->AddPartition(i);
-
- TAutoPtr<NBus::TBusMessage> reply;
- NBus::EMessageStatus status = server.AnnoyingClient->SyncCall(request, reply);
- UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK);
- const NMsgBusProxy::TBusResponse* response = dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Get());
- UNIT_ASSERT(response);
- if (response->Record.GetStatus() == NMsgBusProxy::MSTATUS_OK)
- break;
- UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
- Sleep(TDuration::MilliSeconds(10));
- }
- }
-
- Y_UNIT_TEST(WhenDisableNodeAndCreateTopic_ThenAllPartitionsAreOnOtherNode) {
- // Arrange
- TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::HIVE });
- TString unused = "rt3.dc1--unusedtopic";
- // Just to make sure that HIVE has started
- server.AnnoyingClient->CreateTopic(unused, 1);
- WaitResolveSuccess(server, unused, 1);
-
- // Act
- // Disable node #0
- server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- WaitResolveSuccess(server, DEFAULT_TOPIC_NAME, 3);
-
- // Assert that all partitions are on node #1
- const ui32 node1Id = server.CleverServer->GetRuntime()->GetNodeId(1);
- UNIT_ASSERT_VALUES_EQUAL(
- server.AnnoyingClient->GetPartLocation({{DEFAULT_TOPIC_NAME, {0, 1}}}, 2, true),
- TVector<ui32>({node1Id, node1Id})
- );
- }
-
- void PrepareForGrpc(TTestServer& server) {
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->InitUserRegistry();
- }
- void PrepareForFstClass(TFlatMsgBusPQClient& annoyingClient) {
- annoyingClient.SetNoConfigMode();
- annoyingClient.FullInit({});
- annoyingClient.InitUserRegistry();
- annoyingClient.MkDir("/Root", "account1");
- annoyingClient.CreateTopicNoLegacy(FC_TOPIC_PATH, 5);
- }
-
- Y_UNIT_TEST(CheckACLForGrpcWrite) {
- TTestServer server;
- PrepareForGrpc(server);
-
- auto writer = MakeDataWriter(server, "source1");
-
- writer.Write(SHORT_TOPIC_NAME, "valuevaluevalue1", true, "topic1@" BUILTIN_ACL_DOMAIN);
-
- NACLib::TDiffACL acl;
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", DEFAULT_TOPIC_NAME, acl.SerializeAsString());
-
- Sleep(TDuration::Seconds(2));
-
- auto writer2 = MakeDataWriter(server, "source1");
- writer2.Write(SHORT_TOPIC_NAME, "valuevaluevalue1", false, "topic1@" BUILTIN_ACL_DOMAIN);
- }
-
- void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) {
- annoyingClient.SetNoConfigMode();
- annoyingClient.FullInit();
- annoyingClient.InitUserRegistry();
- annoyingClient.MkDir("/Root", "account1");
- annoyingClient.MkDir("/Root/PQ", "account1");
- annoyingClient.CreateTopicNoLegacy("/Root/PQ/rt3.db--topic1", 5, false);
- annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false, true, {}, {"user1", "user2"});
- annoyingClient.CreateTopicNoLegacy("/Root/account2/topic2", 5);
- }
-
- Y_UNIT_TEST(TestGrpcWriteNoDC) {
- TTestServer server(false);
- server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- server.ServerSettings.PQConfig.SetRoot("/Rt2/PQ");
- server.ServerSettings.PQConfig.SetDatabase("/Root");
- server.StartServer();
-
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"folder_id", "somefolder"}, {"cloud_id", "somecloud"}, {"database_id", "root"}}));
-
-
- PrepareForGrpcNoDC(*server.AnnoyingClient);
- auto writer = MakeDataWriter(server, "source1");
-
- writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
-
- NACLib::TDiffACL acl;
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN);
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "topic1@" BUILTIN_ACL_DOMAIN);
-
- server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString());
- server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString());
-
- Sleep(TDuration::Seconds(5));
-
- writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
-
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
- writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
-
- writer.Read("/Root/PQ/account1/topic1", "user1", "topic1@" BUILTIN_ACL_DOMAIN, false, false, false, true);
-
- writer.Write("Root/PQ/account1/topic1", {"valuevaluevalue2"}, false, "topic1@" BUILTIN_ACL_DOMAIN); //TODO /Root remove
-
- writer.Read("Root/PQ/account1/topic1", "user1", "topic1@" BUILTIN_ACL_DOMAIN, false, false, false, true);
- }
-
- Y_UNIT_TEST(CheckACLForGrpcRead) {
- TTestServer server(PQSettings(0, 1));
- PrepareForGrpc(server);
- TString topic2 = "rt3.dc1--topic2";
- TString topic2ShortName = "topic2";
- server.AnnoyingClient->CreateTopic(
- topic2, 1, 8_MB, 86400, 20000000, "", 200000000, {"user1", "user2"}
- );
- server.EnableLogs({NKikimrServices::PERSQUEUE}, NActors::NLog::PRI_INFO);
-
- server.AnnoyingClient->CreateConsumer("user1");
- server.AnnoyingClient->CreateConsumer("user2");
- server.AnnoyingClient->CreateConsumer("user5");
- server.AnnoyingClient->GrantConsumerAccess("user1", "user2@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->GrantConsumerAccess("user1", "user3@" BUILTIN_ACL_DOMAIN);
-
- server.AnnoyingClient->GrantConsumerAccess("user1", "1@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->GrantConsumerAccess("user2", "2@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->GrantConsumerAccess("user5", "1@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->GrantConsumerAccess("user5", "2@" BUILTIN_ACL_DOMAIN);
-
- auto writer = MakeDataWriter(server, "source1");
-
- NACLib::TDiffACL acl;
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "1@" BUILTIN_ACL_DOMAIN);
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "2@" BUILTIN_ACL_DOMAIN);
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user1@" BUILTIN_ACL_DOMAIN);
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user2@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
-
- Sleep(TDuration::Seconds(2));
-
- auto ticket1 = "1@" BUILTIN_ACL_DOMAIN;
- auto ticket2 = "2@" BUILTIN_ACL_DOMAIN;
-
- writer.Read(topic2ShortName, "user1", ticket1, false, false, false, true);
-
- writer.Read(topic2ShortName, "user1", "user2@" BUILTIN_ACL_DOMAIN, false, false, false, true);
- writer.Read(topic2ShortName, "user1", "user3@" BUILTIN_ACL_DOMAIN, true, false, false, true); //for topic
- writer.Read(topic2ShortName, "user1", "user1@" BUILTIN_ACL_DOMAIN, true, false, false, true); //for consumer
-
- writer.Read(topic2ShortName, "user2", ticket1, true, false, false, true);
- writer.Read(topic2ShortName, "user2", ticket2, false, false, false, true);
-
- writer.Read(topic2ShortName, "user5", ticket1, true, false, false, true);
- writer.Read(topic2ShortName, "user5", ticket2, true, false, false, true);
-
- acl.Clear();
- acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user3@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
-
- Sleep(TDuration::Seconds(2));
- writer.Read(topic2ShortName, "user1", "user3@" BUILTIN_ACL_DOMAIN, false, true, false, true);
- }
-
-
-
- Y_UNIT_TEST(CheckKillBalancer) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY});
- server.EnableLogs({ NKikimrServices::PERSQUEUE}, NActors::NLog::PRI_INFO);
-
- PrepareForGrpc(server);
-
- auto writer = MakeDataWriter(server, "source1");
- TTestPQLib PQLib(server);
- auto [consumer, p] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "shared/user", {}, true);
- Cerr << p.Response << "\n";
-
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_LOCK);
- auto pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo());
-
- msg = consumer->GetNextMessage();
- msg.Wait();
-
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_LOCK);
- pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo());
-
- msg = consumer->GetNextMessage();
-
- //TODO: make here infly commits - check results
-
- server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
- Cerr << "Balancer killed\n";
-
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_RELEASE);
- UNIT_ASSERT(!msg.GetValue().Response.GetRelease().GetCanCommit());
-
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_RELEASE);
- UNIT_ASSERT(!msg.GetValue().Response.GetRelease().GetCanCommit());
-
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_LOCK);
-
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_LOCK);
-
-
- msg = consumer->GetNextMessage();
- bool res = msg.Wait(TDuration::Seconds(1));
- Y_VERIFY(!res); //no read signalled or lock signals
- }
-
- Y_UNIT_TEST(TestWriteStat) {
- auto testWriteStat = [](const TString& originallyProvidedConsumerName,
- const TString& consumerName,
- const TString& consumerPath) {
- auto checkCounters = [](auto monPort, const TString& session,
- const std::set<std::string>& canonicalSensorNames,
- const TString& clientDc, const TString& originDc,
- const TString& client, const TString& consumerPath) {
- NJson::TJsonValue counters;
- if (clientDc.empty() && originDc.empty()) {
- counters = GetClientCountersLegacy(monPort, "pqproxy", session, client, consumerPath);
- } else {
- counters = GetCountersLegacy(monPort, "pqproxy", session, "account/topic1",
- clientDc, originDc, client, consumerPath);
- }
- const auto sensors = counters["sensors"].GetArray();
- std::set<std::string> sensorNames;
- std::transform(sensors.begin(), sensors.end(),
- std::inserter(sensorNames, sensorNames.begin()),
- [](auto& el) {
- return el["labels"]["sensor"].GetString();
- });
- auto equal = sensorNames == canonicalSensorNames;
- UNIT_ASSERT(equal);
- };
-
- auto settings = PQSettings(0, 1, true, "10");
- TTestServer server(settings, false);
- server.PrepareNetDataFile(FormNetData());
- server.StartServer();
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER });
-
- const auto monPort = TPortManager().GetPort();
- auto Counters = server.CleverServer->GetGRpcServerRootCounters();
- NActors::TMon Monitoring({monPort, "localhost", 3, "root", "localhost", {}, {}, {}});
- Monitoring.RegisterCountersPage("counters", "Counters", Counters);
- Monitoring.Start();
-
- auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor();
-
- GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing
-
- server.AnnoyingClient->CreateTopic("rt3.dc1--account--topic1", 10, 10000, 10000, 2000);
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
- auto [producer, res] = PQLib.CreateProducer("account--topic1", "base64:AAAAaaaa____----12", {}, ECodec::RAW);
-
- Cerr << res.Response << "\n";
- TInstant st(TInstant::Now());
- for (ui32 i = 1; i <= 5; ++i) {
- auto f = producer->Write(i, TString(2000, 'a'));
- f.Wait();
- Cerr << f.GetValue().Response << "\n";
- if (i == 5) {
- UNIT_ASSERT(TInstant::Now() - st > TDuration::Seconds(3));
- UNIT_ASSERT(f.GetValue().Response.GetAck().GetStat().GetPartitionQuotedTimeMs() <=
- f.GetValue().Response.GetAck().GetStat().GetTotalTimeInPartitionQueueMs() + 100);
- }
- }
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "writeSession",
- {
- "BytesWrittenOriginal",
- "CompactedBytesWrittenOriginal",
- "MessagesWrittenOriginal",
- "UncompressedBytesWrittenOriginal"
- },
- "", "cluster", "", ""
- );
-
- checkCounters(monPort,
- "writeSession",
- {
- "BytesInflight",
- "BytesInflightTotal",
- "Errors",
- "SessionsActive",
- "SessionsCreated",
- "WithoutAuth"
- },
- "", "cluster", "", ""
- );
- {
- auto [consumer, res] = PQLib.CreateConsumer({"account/topic1"}, originallyProvidedConsumerName,
- {}, false);
- Cerr << res.Response << "\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_DATA);
-
- checkCounters(monPort,
- "readSession",
- {
- "Commits",
- "PartitionsErrors",
- "PartitionsInfly",
- "PartitionsLocked",
- "PartitionsReleased",
- "PartitionsToBeLocked",
- "PartitionsToBeReleased",
- "WaitsForData"
- },
- "", "cluster", "", ""
- );
-
- checkCounters(monPort,
- "readSession",
- {
- "BytesInflight",
- "Errors",
- "PipeReconnects",
- "SessionsActive",
- "SessionsCreated",
- "PartsPerSession",
- "SessionsWithOldBatchingVersion",
- "WithoutAuth"
- },
- "", "", consumerName, consumerPath
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "readSession",
- {
- "BytesReadFromDC"
- },
- "Vla", "", "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "readSession",
- {
- "BytesRead",
- "MessagesRead"
- },
- "", "Dc1", "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "readSession",
- {
- "BytesRead",
- "MessagesRead"
- },
- "", "cluster", "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "readSession",
- {
- "BytesRead",
- "MessagesRead"
- },
- "", "cluster", "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- "readSession",
- {
- "BytesRead",
- "MessagesRead"
- },
- "", "Dc1", consumerName, consumerPath
- );
- }
- };
-
- testWriteStat("some@random@consumer", "some@random@consumer", "some/random/consumer");
- testWriteStat("some@user", "some@user", "some/user");
- testWriteStat("shared@user", "shared@user", "shared/user");
- testWriteStat("shared/user", "user", "shared/user");
- testWriteStat("user", "user", "shared/user");
- testWriteStat("some@random/consumer", "some@random@consumer", "some/random/consumer");
- testWriteStat("/some/user", "some@user", "some/user");
- }
-
-
- Y_UNIT_TEST(TestWriteStat1stClass) {
- auto testWriteStat1stClass = [](const TString& consumerPath) {
- const auto folderId = "somefolder";
- const auto cloudId = "somecloud";
- const auto databaseId = "root";
- const TString fullTopicName{"/Root/account/topic1"};
- const TString topicName{"account/topic1"};
-
- auto checkCounters =
- [cloudId, folderId, databaseId](auto monPort,
- const std::set<std::string>& canonicalSensorNames,
- const TString& stream, const TString& consumer,
- const TString& host, const TString& shard) {
- auto counters = GetCounters1stClass(monPort, "datastreams", cloudId, databaseId,
- folderId, stream, consumer, host, shard);
- const auto sensors = counters["sensors"].GetArray();
- std::set<std::string> sensorNames;
- std::transform(sensors.begin(), sensors.end(),
- std::inserter(sensorNames, sensorNames.begin()),
- [](auto& el) {
- return el["labels"]["name"].GetString();
- });
- auto equal = sensorNames == canonicalSensorNames;
- UNIT_ASSERT(equal);
- };
-
- auto settings = PQSettings(0, 1, true, "10");
- TTestServer server(settings, false);
- server.PrepareNetDataFile(FormNetData());
- server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- server.StartServer();
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER });
-
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- server.AnnoyingClient->AlterUserAttributes("/", "Root",
- {{"folder_id", folderId},
- {"cloud_id", cloudId},
- {"database_id", databaseId}}));
-
- const auto monPort = TPortManager().GetPort();
- auto Counters = server.CleverServer->GetGRpcServerRootCounters();
- NActors::TMon Monitoring({monPort, "localhost", 3, "root", "localhost", {}, {}, {}});
- Monitoring.RegisterCountersPage("counters", "Counters", Counters);
- Monitoring.Start();
-
- auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor();
-
- GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing
-
- server.AnnoyingClient->SetNoConfigMode();
- server.AnnoyingClient->FullInit();
- server.AnnoyingClient->InitUserRegistry();
- server.AnnoyingClient->MkDir("/Root", "account");
- server.AnnoyingClient->CreateTopicNoLegacy(fullTopicName, 5);
-
- NYdb::TDriverConfig driverCfg;
- driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort)
- .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root");
-
- auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg);
- auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver);
-
- {
- auto res = persQueueClient->AddReadRule(fullTopicName,
- NYdb::NPersQueue::TAddReadRuleSettings().ReadRule(NYdb::NPersQueue::TReadRuleSettings().ConsumerName(consumerPath)));
- res.Wait();
- UNIT_ASSERT(res.GetValue().IsSuccess());
- }
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
- auto [producer, res] = PQLib.CreateProducer(fullTopicName, "base64:AAAAaaaa____----12", {},
- ECodec::RAW);
-
- Cerr << res.Response << "\n";
- for (ui32 i = 1; i <= 5; ++i) {
- auto f = producer->Write(i, TString(2000, 'a'));
- f.Wait();
- Cerr << f.GetValue().Response << "\n";
-
- }
-
- {
- auto [consumer, res] = PQLib.CreateConsumer({topicName}, consumerPath, {},
- false);
- Cerr << res.Response << "\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
-
- checkCounters(monPort,
- {
- "stream.internal_read.commits_per_second",
- "stream.internal_read.partitions_errors_per_second",
- "stream.internal_read.partitions_locked",
- "stream.internal_read.partitions_locked_per_second",
- "stream.internal_read.partitions_released_per_second",
- "stream.internal_read.partitions_to_be_locked",
- "stream.internal_read.partitions_to_be_released",
- "stream.internal_read.waits_for_data",
- "stream.internal_write.bytes_proceeding",
- "stream.internal_write.bytes_proceeding_total",
- "stream.internal_write.errors_per_second",
- "stream.internal_write.sessions_active",
- "stream.internal_write.sessions_created_per_second",
- "stream.internal_write.sessions_without_auth"
- },
- topicName, "", "", ""
- );
-
- checkCounters(monPort,
- {
- "stream.internal_read.commits_per_second",
- "stream.internal_read.partitions_errors_per_second",
- "stream.internal_read.partitions_locked",
- "stream.internal_read.partitions_locked_per_second",
- "stream.internal_read.partitions_released_per_second",
- "stream.internal_read.partitions_to_be_locked",
- "stream.internal_read.partitions_to_be_released",
- "stream.internal_read.waits_for_data",
- },
- topicName, consumerPath, "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- {
- "stream.internal_read.time_lags_milliseconds",
- "stream.incoming_bytes_per_second",
- "stream.incoming_records_per_second",
- "stream.internal_write.bytes_per_second",
- "stream.internal_write.compacted_bytes_per_second",
- "stream.internal_write.partition_write_quota_wait_milliseconds",
- "stream.internal_write.record_size_bytes",
- "stream.internal_write.records_per_second",
- "stream.internal_write.time_lags_milliseconds",
- "stream.internal_write.uncompressed_bytes_per_second",
- "stream.await_operating_milliseconds",
- "stream.internal_write.buffer_brimmed_duration_ms",
- "stream.internal_read.bytes_per_second",
- "stream.internal_read.records_per_second",
- "stream.outgoing_bytes_per_second",
- "stream.outgoing_records_per_second",
- },
- topicName, "", "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- {
- "stream.internal_read.time_lags_milliseconds",
- "stream.await_operating_milliseconds",
- "stream.internal_read.bytes_per_second",
- "stream.internal_read.records_per_second",
- "stream.outgoing_bytes_per_second",
- "stream.outgoing_records_per_second",
- },
- topicName, consumerPath, "", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- {
- "stream.await_operating_milliseconds"
- },
- topicName, consumerPath, "1", ""
- );
-
- checkCounters(server.CleverServer->GetRuntime()->GetMonPort(),
- {
- "stream.internal_write.buffer_brimmed_duration_ms"
- },
- topicName, "", "1", ""
- );
- }
- };
- testWriteStat1stClass("some@random@consumer");
- testWriteStat1stClass("user1");
- }
-
-
- Y_UNIT_TEST(TestUnorderedCommit) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- for (ui32 i = 1; i <= 3; ++i) {
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123" + ToString<ui32>(i), i, ECodec::RAW);
- UNIT_ASSERT(res.Response.HasInit());
- auto f = producer->Write(i, TString(10, 'a'));
- f.Wait();
- }
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, false, false);
- Cerr << res.Response << "\n";
- for (ui32 i = 1; i <= 3; ++i) {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasData());
- UNIT_ASSERT(msg.GetValue().Response.GetData().GetCookie() == i);
- }
- auto msg = consumer->GetNextMessage();
- consumer->Commit({2});
- UNIT_ASSERT(!msg.Wait(TDuration::Seconds(1)));
- consumer->Commit({1});
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasCommit());
- UNIT_ASSERT(msg.GetValue().Response.GetCommit().CookieSize() == 2);
- UNIT_ASSERT(msg.GetValue().Response.GetCommit().GetCookie(0) == 1);
- UNIT_ASSERT(msg.GetValue().Response.GetCommit().GetCookie(1) == 2);
- consumer->Commit({3});
- msg = consumer->GetNextMessage();
- UNIT_ASSERT(!msg.Wait(TDuration::MilliSeconds(500)));
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasCommit());
- UNIT_ASSERT(msg.GetValue().Response.GetCommit().CookieSize() == 1);
- UNIT_ASSERT(msg.GetValue().Response.GetCommit().GetCookie(0) == 3);
- consumer->Commit({4}); //not existed cookie
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasError());
- }
-
- Y_UNIT_TEST(TestMaxReadCookies) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto writer = MakeDataWriter(server);
-
- TTestPQLib PQLib(server);
- {
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- for (ui32 i = 1; i <= 11; ++i) {
- auto f = producer->Write(i, TString(10, 'a'));
- f.Wait();
- }
- }
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, false);
- Cerr << res.Response << "\n";
- for (ui32 i = 1; i <= 11; ++i) {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasData());
- UNIT_ASSERT(msg.GetValue().Response.GetData().GetCookie() == i);
- }
- for (ui32 i = 11; i >= 1; --i) {
- consumer->Commit({i});
- if (i == 5) {
- server.AnnoyingClient->GetClientInfo({DEFAULT_TOPIC_NAME}, "user", true);
- }
- }
- Cerr << "cookies committed\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasError());
- }
-
- Y_UNIT_TEST(TestWriteSessionsConflicts) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- auto writer = MakeDataWriter(server);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- TTestPQLib PQLib(server);
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- auto dead = producer->IsDead();
-
- auto [producer2, res2] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
-
- UNIT_ASSERT(dead.Wait(TDuration::Seconds(1)));
- UNIT_ASSERT(!producer2->IsDead().Wait(TDuration::Seconds(1)));
- }
-
- Y_UNIT_TEST(TestWriteSessionNaming) {
- TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
-
- TTestPQLib PQLib(server);
- while (true) {
- auto [producer, res] = PQLib.CreateProducer("//", "123");
- Cerr << res.Response << "\n";
- UNIT_ASSERT(res.Response.HasError());
- if (res.Response.GetError().GetCode() != NPersQueue::NErrorCode::INITIALIZING)
- break;
- Sleep(TDuration::Seconds(1));
- }
- }
-
-
- Y_UNIT_TEST(TestRelocks) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
-
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
-
- TIntrusivePtr<TCerrLogger> logger = new TCerrLogger(DEBUG_LOG_LEVEL);
- TPQLib PQLib;
-
- TConsumerSettings ss;
- ss.ClientId = "user";
- ss.Server = TServerSetting{"localhost", server.GrpcPort};
- ss.Topics.push_back(SHORT_TOPIC_NAME);
- ss.UseLockSession = true;
- ss.MaxCount = 1;
-
- auto consumer = PQLib.CreateConsumer(ss, logger, false);
- auto p = consumer->Start();
- for (ui32 i = 0; i < 30; ++i) {
- server.CleverServer->GetRuntime()->ResetScheduledCount();
- server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
- Sleep(TDuration::MilliSeconds(1));
- }
- p.Wait();
- Cerr << p.GetValue().Response << "\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- }
-
- Y_UNIT_TEST(TestLockErrors) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
-
- TTestPQLib PQLib(server);
-
- {
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- for (ui32 i = 1; i <= 11; ++i) {
- auto f = producer->Write(i, TString(10, 'a'));
- f.Wait();
- }
- }
- auto createConsumer = [&] () {
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, true);
- return std::move(consumer);
- };
- auto consumer = createConsumer();
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetReadOffset() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetEndOffset() == 11);
-
- auto pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo{0, 5, false});
- auto future = consumer->IsDead();
- future.Wait();
- Cerr << future.GetValue() << "\n";
-
-
- consumer = createConsumer();
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetReadOffset() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetEndOffset() == 11);
-
- pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo{12, 12, false});
- future = consumer->IsDead();
- future.Wait();
- Cerr << future.GetValue() << "\n";
-
- consumer = createConsumer();
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetReadOffset() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetEndOffset() == 11);
-
- pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo{11, 11, false});
- Sleep(TDuration::Seconds(5));
-
-
- consumer = createConsumer();
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetReadOffset() == 11);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetEndOffset() == 11);
-
- pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo{1, 0, true});
- future = consumer->IsDead();
- future.Wait();
- Cerr << future.GetValue() << "\n";
-
- consumer = createConsumer();
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetReadOffset() == 11);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetEndOffset() == 11);
-
- pp = msg.GetValue().ReadyToRead;
- pp.SetValue(TLockInfo{0, 0, false});
- future = consumer->IsDead();
- UNIT_ASSERT(!future.Wait(TDuration::Seconds(5)));
- }
-
-
- Y_UNIT_TEST(TestLocalChoose) {
- TTestServer server(false);
- server.ServerSettings.NodeCount = 3;
- server.StartServer();
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
-
- server.EnableLogs({ NKikimrServices::CHOOSE_PROXY });
-
- auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
- auto stub(NKikimrClient::TGRpcServer::NewStub(channel));
-
- auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor();
-
- auto nodeId = server.CleverServer->GetRuntime()->GetNodeId(0);
- server.CleverServer->GetRuntime()->Send(new IEventHandle(
- MakeGRpcProxyStatusID(nodeId), sender, new TEvGRpcProxyStatus::TEvSetup(true, 0, 0)), 0, false
- );
- nodeId = server.CleverServer->GetRuntime()->GetNodeId(1);
- server.CleverServer->GetRuntime()->Send(
- new IEventHandle(
- MakeGRpcProxyStatusID(nodeId), sender, new TEvGRpcProxyStatus::TEvSetup(true, 10000000, 1000000)
- ), 1, false
- );
- nodeId = server.CleverServer->GetRuntime()->GetNodeId(2);
- server.CleverServer->GetRuntime()->Send(
- new IEventHandle(
- MakeGRpcProxyStatusID(nodeId), sender, new TEvGRpcProxyStatus::TEvSetup(true, 10000000, 1000000)
- ), 2, false
- );
-
- grpc::ClientContext context;
- NKikimrClient::TChooseProxyRequest request;
- request.SetPreferLocalProxy(true);
- NKikimrClient::TResponse response;
- auto status = stub->ChooseProxy(&context, request, &response);
- UNIT_ASSERT(status.ok());
- Cerr << response << "\n";
- UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- UNIT_ASSERT(response.GetProxyCookie() == server.CleverServer->GetRuntime()->GetNodeId(0));
-
- grpc::ClientContext context2;
- request.SetPreferLocalProxy(false);
- NKikimrClient::TResponse response2;
- status = stub->ChooseProxy(&context2, request, &response2);
- UNIT_ASSERT(status.ok());
- Cerr << response2 << "\n";
- UNIT_ASSERT(response2.GetStatus() == NMsgBusProxy::MSTATUS_OK);
- UNIT_ASSERT(response2.GetProxyCookie() > server.CleverServer->GetRuntime()->GetNodeId(0));
- }
-
-
- Y_UNIT_TEST(TestRestartBalancer) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- {
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- for (ui32 i = 1; i < 2; ++i) {
- auto f = producer->Write(i, TString(10, 'a'));
- f.Wait();
- }
- }
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, true, {}, {}, 100, {});
- Cerr << res.Response << "\n";
-
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME);
- UNIT_ASSERT(msg.GetValue().Response.GetLock().GetPartition() == 0);
-
-
- msg.GetValue().ReadyToRead.SetValue(TLockInfo{0,0,false});
-
- msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasData());
-// Sleep(TDuration::MilliSeconds(10));
- server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
-
- msg = consumer->GetNextMessage();
- msg.Wait();
-
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT_C(msg.GetValue().Response.HasRelease(), "Response: " << msg.GetValue().Response);
- }
-
- Y_UNIT_TEST(TestBigMessage) {
- TTestServer server(false);
- server.GrpcServerOptions.SetMaxMessageSize(130_MB);
- server.StartServer();
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- auto f = producer->Write(1, TString(63_MB, 'a'));
- f.Wait();
- Cerr << f.GetValue().Response << "\n";
- UNIT_ASSERT_C(f.GetValue().Response.HasAck(), f.GetValue().Response);
- }
-
- void TestRereadsWhenDataIsEmptyImpl(bool withWait) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- // Write nonempty data
- NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4);
-
- auto write = [&](const TString& data, bool empty = false) {
- NKikimrPQClient::TDataChunk dataChunk;
- dataChunk.SetCreateTime(42);
- dataChunk.SetSeqNo(++writeReq.SeqNo);
- dataChunk.SetData(data);
- if (empty) {
- dataChunk.SetChunkType(NKikimrPQClient::TDataChunk::GROW); // this guarantees that data will be threated as empty
- }
- TString serialized;
- UNIT_ASSERT(dataChunk.SerializeToString(&serialized));
- server.AnnoyingClient->WriteToPQ(writeReq, serialized);
- };
- write("data1");
- write("data2", true);
- if (!withWait) {
- write("data3");
- }
-
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, false, {}, {}, 1, 1);
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
- auto msg1 = consumer->GetNextMessage().GetValueSync().Response;
-
- auto assertHasData = [](const NPersQueue::TReadResponse& msg, const TString& data) {
- const auto& d = msg.GetData();
- UNIT_ASSERT_VALUES_EQUAL_C(d.MessageBatchSize(), 1, msg);
- UNIT_ASSERT_VALUES_EQUAL_C(d.GetMessageBatch(0).MessageSize(), 1, msg);
- UNIT_ASSERT_STRINGS_EQUAL_C(d.GetMessageBatch(0).GetMessage(0).GetData(), data, msg);
- };
- UNIT_ASSERT_VALUES_EQUAL_C(msg1.GetData().GetCookie(), 1, msg1);
- assertHasData(msg1, "data1");
-
- auto resp2Future = consumer->GetNextMessage();
- if (withWait) {
- // no data
- UNIT_ASSERT(!resp2Future.HasValue());
- UNIT_ASSERT(!resp2Future.HasException());
-
- // waits and data doesn't arrive
- Sleep(TDuration::MilliSeconds(100));
- UNIT_ASSERT(!resp2Future.HasValue());
- UNIT_ASSERT(!resp2Future.HasException());
-
- // write data
- write("data3");
- }
- const auto& msg2 = resp2Future.GetValueSync().Response;
- UNIT_ASSERT_VALUES_EQUAL_C(msg2.GetData().GetCookie(), 2, msg2);
- assertHasData(msg2, "data3");
- }
-
- Y_UNIT_TEST(TestRereadsWhenDataIsEmpty) {
- TestRereadsWhenDataIsEmptyImpl(false);
- }
-
- Y_UNIT_TEST(TestRereadsWhenDataIsEmptyWithWait) {
- TestRereadsWhenDataIsEmptyImpl(true);
- }
-
-
- Y_UNIT_TEST(TestLockAfterDrop) {
- TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- auto f = producer->Write(1, TString(1_KB, 'a'));
- f.Wait();
-
- auto [consumer, res2] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, true);
- Cerr << res2.Response << "\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- UNIT_ASSERT_C(msg.GetValue().Response.HasLock(), msg.GetValue().Response);
- UNIT_ASSERT_C(msg.GetValue().Response.GetLock().GetTopic() == DEFAULT_TOPIC_NAME, msg.GetValue().Response);
- UNIT_ASSERT_C(msg.GetValue().Response.GetLock().GetPartition() == 0, msg.GetValue().Response);
-
- server.CleverServer->GetRuntime()->ResetScheduledCount();
- server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
-
- msg.GetValue().ReadyToRead.SetValue({0,0,false});
-
- msg = consumer->GetNextMessage();
- UNIT_ASSERT(msg.Wait(TDuration::Seconds(10)));
-
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasData());
- }
-
-
- Y_UNIT_TEST(TestMaxNewTopicModel) {
- TTestServer server;
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}});
- server.AnnoyingClient->CreateTopic("rt3.dc1--aaa@bbb@ccc--topic", 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
- {
- auto [producer, res] = PQLib.CreateProducer("aaa/bbb/ccc/topic", "123");
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
- for (ui32 i = 1; i <= 11; ++i) {
- auto f = producer->Write(i, TString(10, 'a'));
- f.Wait();
- UNIT_ASSERT_C(f.GetValue().Response.HasAck(), f.GetValue().Response);
- }
- }
-
- auto [consumer, res2] = PQLib.CreateConsumer({"aaa/bbb/ccc/topic"}, "user", 1, true);
- UNIT_ASSERT_C(res2.Response.HasInit(), res2.Response);
-
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Response.HasLock());
- }
-
- Y_UNIT_TEST(TestPartitionStatus) {
- TTestServer server;
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}});
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, true);
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
-
- auto msg = consumer->GetNextMessage();
- auto value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasLock());
- value.ReadyToRead.SetValue(TLockInfo{});
- auto lock = value.Response.GetLock();
- consumer->RequestPartitionStatus(lock.GetTopic(), lock.GetPartition(), lock.GetGeneration());
- msg = consumer->GetNextMessage();
- value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasPartitionStatus());
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetCommittedOffset() == 0);
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetEndOffset() == 0);
- auto wt = value.Response.GetPartitionStatus().GetWriteWatermarkMs();
- Sleep(TDuration::Seconds(15));
-
- consumer->RequestPartitionStatus(lock.GetTopic(), lock.GetPartition(), lock.GetGeneration());
- msg = consumer->GetNextMessage();
- value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasPartitionStatus());
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetCommittedOffset() == 0);
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetEndOffset() == 0);
- UNIT_ASSERT(wt < value.Response.GetPartitionStatus().GetWriteWatermarkMs());
- wt = value.Response.GetPartitionStatus().GetWriteWatermarkMs();
-
- {
- auto [producer, res] = PQLib.CreateProducer(SHORT_TOPIC_NAME, "123");
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
- auto f = producer->Write(1, TString(10, 'a'));
- UNIT_ASSERT_C(f.GetValueSync().Response.HasAck(), f.GetValueSync().Response);
- }
- msg = consumer->GetNextMessage();
- value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasData());
- auto cookie = value.Response.GetData().GetCookie();
-
- consumer->RequestPartitionStatus(lock.GetTopic(), lock.GetPartition(), lock.GetGeneration());
- msg = consumer->GetNextMessage();
- value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasPartitionStatus());
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetCommittedOffset() == 0);
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetEndOffset() == 1);
- UNIT_ASSERT(wt < value.Response.GetPartitionStatus().GetWriteWatermarkMs());
- wt = value.Response.GetPartitionStatus().GetWriteWatermarkMs();
- consumer->Commit({cookie});
- msg = consumer->GetNextMessage();
- Cerr << msg.GetValueSync().Response << "\n";
- UNIT_ASSERT(msg.GetValueSync().Response.HasCommit());
-
- consumer->RequestPartitionStatus(lock.GetTopic(), lock.GetPartition(), lock.GetGeneration());
- msg = consumer->GetNextMessage();
- value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasPartitionStatus());
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetCommittedOffset() == 1);
- UNIT_ASSERT(value.Response.GetPartitionStatus().GetEndOffset() == 1);
- }
-
- Y_UNIT_TEST(TestDeletionOfTopic) {
- TTestServer server(false);
- server.GrpcServerOptions.SetMaxMessageSize(130_MB);
- server.StartServer();
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
-
- auto writer = MakeDataWriter(server);
- TTestPQLib PQLib(server);
-
- server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME});
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false);
- Sleep(TDuration::Seconds(1));
- auto [consumer, res] = PQLib.CreateConsumer({SHORT_TOPIC_NAME}, "user", 1, {}, {}, false);
- Cerr << res.Response << "\n";
-
- UNIT_ASSERT_EQUAL_C(res.Response.GetError().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC, res.Response);
- }
-
- Y_UNIT_TEST(SetupYqlTimeout) {
- TTestServer server(PQSettings(0, 1, true, "1"));
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
-
- auto writer = MakeDataWriter(server);
-
- server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false);
- server.AnnoyingClient->KickNodeInHive(server.CleverServer->GetRuntime(), 0);
-
- writer.InitSession("sid1", 2, false);
- }
-
- Y_UNIT_TEST(TestFirstClassWriteAndRead) {
- auto settings = PQSettings(0, 1, true, "1");
- settings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- TTestServer server(settings, false);
- server.StartServer(false);
- UNIT_ASSERT_VALUES_EQUAL(NMsgBusProxy::MSTATUS_OK,
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"folder_id", "somefolder"}, {"cloud_id", "somecloud"}, {"database_id", "root"}}));
-
-
- Cerr << "HERE\n";
-
- PrepareForFstClass(*server.AnnoyingClient);
- server.EnableLogs({ NKikimrServices::PQ_METACACHE, NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
- server.EnableLogs({ NKikimrServices::PERSQUEUE_CLUSTER_TRACKER }, NActors::NLog::PRI_EMERG);
- server.EnableLogs({ NKikimrServices::PERSQUEUE_READ_BALANCER }, NActors::NLog::PRI_DEBUG);
-
- TTestPQLib pqLib(server);
-
- NACLib::TDiffACL acl;
-// acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic@" BUILTIN_ACL_DOMAIN);
-// server.AnnoyingClient->ModifyACL("/Root/account1", "root-acc-topic", acl.SerializeAsString());
- Sleep(TDuration::Seconds(5));
- {
- auto [producer, res] = pqLib.CreateProducer(FC_TOPIC_PATH, "123", {}, {}, {}, true);
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
- Cerr << "Producer 1 start response: " << res.Response << "\n";
- auto f = producer->Write(1, TString(10, 'a'));
- UNIT_ASSERT_C(f.GetValueSync().Response.HasAck(), f.GetValueSync().Response);
- }
- {
- auto [producer, res] = pqLib.CreateProducer(FC_TOPIC_PATH, "123", {}, {}, {}, true);
- UNIT_ASSERT_C(res.Response.HasInit(), res.Response);
- Cerr << "Producer 2 start response: " << res.Response << "\n";
- auto f = producer->Write(2, TString(10, 'b'));
- UNIT_ASSERT_C(f.GetValueSync().Response.HasAck(), f.GetValueSync().Response);
- }
- {
- auto [consumer, res] = pqLib.CreateConsumer({FC_TOPIC_PATH}, "user");
- Cerr << "Consumer start response: " << res.Response << "\n";
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- Cerr << "Read response: " << msg.GetValue().Response << "\n";
- UNIT_ASSERT(msg.GetValue().Type == EMT_DATA);
- }
- {
- auto [consumer, res] = pqLib.CreateConsumer({FC_TOPIC_PATH}, "user", {}, true);
- Cerr << "Consumer start response: " << res.Response << "\n";
- ui64 commitsDone = 0;
- while (true) {
- auto msg = consumer->GetNextMessage();
- msg.Wait();
- auto& value = msg.GetValue();
- if (value.Type == EMT_LOCK) {
- TStringBuilder message;
- Cerr << "Consumer lock response: " << value.Response << "\n";
- UNIT_ASSERT_VALUES_EQUAL(value.Response.GetLock().GetTopic(), "account1/root-acc-topic");
- msg.GetValue().ReadyToRead.SetValue(TLockInfo());
- } else if (value.Type == EMT_DATA) {
- auto cookie = msg.GetValue().Response.GetData().GetCookie();
- consumer->Commit({cookie});
- } else {
- UNIT_ASSERT(value.Type == EMT_COMMIT);
- commitsDone++;
- break;
- }
- }
- UNIT_ASSERT(commitsDone > 0);
- }
- }
-
- Y_UNIT_TEST(SrcIdCompatibility) {
- TString srcId1 = "test-src-id-compat", srcId2 = "test-src-id-compat2";
- TTestServer server{};
- TString topicName = "rt3.dc1--topic100";
- TString fullPath = "Root/PQ/rt3.dc1--topic100";
- TString shortTopicName = "topic100";
- server.AnnoyingClient->CreateTopic(topicName, 100);
- server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
-
- auto runTest = [&] (
- const TString& topicToAdd, const TString& topicForHash, const TString& topicName,
- const TString& srcId, ui32 partId, ui64 accessTime = 0
- ) {
- TStringBuilder query;
- auto encoded = NPQ::NSourceIdEncoding::EncodeSrcId(topicForHash, srcId);
- Cerr << "===save partition with time: " << accessTime << Endl;
-
- if (accessTime == 0) {
- accessTime = TInstant::Now().MilliSeconds();
- }
- if (!topicToAdd.empty()) { // Empty means don't add anything
- query <<
- "--!syntax_v1\n"
- "UPSERT INTO `/Root/PQ/SourceIdMeta2` (Hash, Topic, SourceId, CreateTime, AccessTime, Partition) VALUES ("
- << encoded.Hash << ", \"" << topicToAdd << "\", \"" << encoded.EscapedSourceId << "\", "
- << TInstant::Now().MilliSeconds() << ", " << accessTime << ", " << partId << "); ";
- Cerr << "Run query:\n" << query << Endl;
- auto scResult = server.AnnoyingClient->RunYqlDataQuery(query);
- //UNIT_ASSERT(scResult.Defined());
- }
- TTestPQLib pqLib(server);
- auto[producer, response] = pqLib.CreateProducer(topicName, srcId, {}, {}, {}, true);
- UNIT_ASSERT_C(response.Response.HasInit(), response.Response);
- UNIT_ASSERT_VALUES_EQUAL(response.Response.GetInit().GetPartition(), partId);
- };
-
- runTest(fullPath, shortTopicName, shortTopicName, srcId1, 5, 100);
- runTest(topicName, shortTopicName, shortTopicName, srcId2, 6);
- runTest("", "", shortTopicName, srcId1, 5, 100);
- // Add another partition to the src mapping with different topic in key.
- // Expect newer partition to be discovered.
- ui64 time = (TInstant::Now() + TDuration::Hours(4)).MilliSeconds();
- runTest(topicName, shortTopicName, shortTopicName, srcId1, 7, time);
-
- }
-
-} // Y_UNIT_TEST_SUITE(TPersQueueTest)
-
-} // namespace NPersQueueTests
-} // namespace NKikimr
diff --git a/kikimr/yndx/grpc_services/persqueue/protocol_compatibility_ut.cpp b/kikimr/yndx/grpc_services/persqueue/protocol_compatibility_ut.cpp
deleted file mode 100644
index f69080ea6b..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/protocol_compatibility_ut.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h>
-#include <ydb/core/client/server/msgbus_server_pq_metacache.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h>
-#include <kikimr/yndx/grpc_services/persqueue/persqueue.h>
-#include <kikimr/yndx/persqueue/msgbus_server/read_session_info.h>
-
-
-namespace NKikimr {
-namespace NPersQueueTests {
-
-Y_UNIT_TEST_SUITE(TPersQueueProtocolCompatibility) {
- Y_UNIT_TEST(GetReadInfoFromV1AboutV0Session) {
- NKikimr::Tests::TServerSettings serverSettings = PQSettings(0);
- serverSettings.RegisterGrpcService<NKikimr::NGRpcService::TGRpcPersQueueService>(
- "pq",
- NKikimr::NMsgBusProxy::CreatePersQueueMetaCacheV2Id()
- );
- serverSettings.SetPersQueueGetReadSessionsInfoWorkerFactory(
- std::make_shared<NKikimr::NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerWithPQv0Factory>()
- );
-
- NPersQueue::TTestServer server(serverSettings);
- server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_READ_PROXY });
- server.AnnoyingClient->CreateTopic("rt3.dc1--topic1", 1);
-
- NPersQueue::TPQLibSettings pqSettings;
- pqSettings.DefaultLogger = new NPersQueue::TCerrLogger(NPersQueue::DEBUG_LOG_LEVEL);
- THolder<NPersQueue::TPQLib> PQLib = MakeHolder<NPersQueue::TPQLib>(pqSettings);
-
- NPersQueue::TConsumerSettings settings;
- settings.Server = NPersQueue::TServerSetting{"localhost", server.GrpcPort};
- settings.ClientId = "user";
- settings.Topics = {"topic1"};
- settings.UseLockSession = true;
- auto consumer = PQLib->CreateConsumer(settings);
- auto response = consumer->Start().GetValueSync();
- UNIT_ASSERT_C(response.Response.HasInit(), response.Response);
-
- auto msg = consumer->GetNextMessage();
- auto value = msg.ExtractValueSync();
- Cerr << value.Response << "\n";
- UNIT_ASSERT(value.Response.HasLock());
- value.ReadyToRead.SetValue(NPersQueue::TLockInfo{});
- auto lock = value.Response.GetLock();
- Cout << lock.DebugString() << Endl;
- {
- std::shared_ptr<grpc::Channel> channel;
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> stub;
-
- {
- channel = grpc::CreateChannel(
- "localhost:" + ToString(server.GrpcPort),
- grpc::InsecureChannelCredentials()
- );
- stub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
- }
- {
- Sleep(TDuration::Seconds(10));
- Ydb::PersQueue::V1::ReadInfoRequest request;
- Ydb::PersQueue::V1::ReadInfoResponse response;
- request.mutable_consumer()->set_path("user");
- request.set_get_only_original(true);
- request.add_topics()->set_path("topic1");
- grpc::ClientContext rcontext;
- auto status = stub->GetReadSessionsInfo(&rcontext, request, &response);
- UNIT_ASSERT(status.ok());
- Ydb::PersQueue::V1::ReadInfoResult res;
- response.operation().result().UnpackTo(&res);
- Cerr << "Read info response: " << response << Endl << res << Endl;
- UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1);
- UNIT_ASSERT(res.topics(0).status() == Ydb::StatusIds::SUCCESS);
- }
- }
-
- }
-
-} // Y_UNIT_TEST_SUITE(TPersQueueProtocolCompatibility)
-
-} // namespace NPersQueueTests
-} // namespace NKikimr
diff --git a/kikimr/yndx/grpc_services/persqueue/ut/definitions.h b/kikimr/yndx/grpc_services/persqueue/ut/definitions.h
deleted file mode 100644
index 35f9f6bc43..0000000000
--- a/kikimr/yndx/grpc_services/persqueue/ut/definitions.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#pragma once
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h>
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-
-namespace NKikimr::NPersQueueTests {
-
-
-namespace {
- const TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1";
- const TString FC_TOPIC_PATH = "/Root/account1/root-acc-topic";
- const TString SHORT_TOPIC_NAME = "topic1";
-}
-
-NPersQueue::NTests::TPQDataWriter MakeDataWriter(NPersQueue::TTestServer& server, const TString& srcId = "source") {
- return NPersQueue::NTests::TPQDataWriter(DEFAULT_TOPIC_NAME, SHORT_TOPIC_NAME, srcId, server);
-}
-
-} // namespace NKikimr::NPersQueueTests
diff --git a/kikimr/yndx/persqueue/msgbus_server/CMakeLists.txt b/kikimr/yndx/persqueue/msgbus_server/CMakeLists.txt
deleted file mode 100644
index c923814a2f..0000000000
--- a/kikimr/yndx/persqueue/msgbus_server/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yndx-persqueue-msgbus_server)
-target_link_libraries(yndx-persqueue-msgbus_server PUBLIC
- contrib-libs-cxxsupp
- yutil
- core-client-server
- yndx-grpc_services-persqueue
-)
-target_sources(yndx-persqueue-msgbus_server PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/persqueue/msgbus_server/read_session_info.cpp
-)
diff --git a/kikimr/yndx/persqueue/msgbus_server/read_session_info.cpp b/kikimr/yndx/persqueue/msgbus_server/read_session_info.cpp
deleted file mode 100644
index 1ccf3b0539..0000000000
--- a/kikimr/yndx/persqueue/msgbus_server/read_session_info.cpp
+++ /dev/null
@@ -1,16 +0,0 @@
-#include "read_session_info.h"
-
-
-namespace NKikimr {
-namespace NMsgBusProxy {
-
-void TPersQueueGetReadSessionsInfoWorkerWithPQv0::SendStatusRequest(const TString& sessionName, TActorId actorId, const TActorContext& ctx) {
- if (sessionName.EndsWith("_v1")) {
- SendStatusRequest<NGRpcProxy::V1::TEvPQProxy::TEvReadSessionStatus>(actorId, ctx);
- } else {
- SendStatusRequest<NGRpcProxy::TEvPQProxy::TEvReadSessionStatus>(actorId, ctx);
- }
-}
-
-} // namespace NMsgBusProxy
-} // namespace NKikimr
diff --git a/kikimr/yndx/persqueue/msgbus_server/read_session_info.h b/kikimr/yndx/persqueue/msgbus_server/read_session_info.h
deleted file mode 100644
index f92572677d..0000000000
--- a/kikimr/yndx/persqueue/msgbus_server/read_session_info.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#pragma once
-
-#include <ydb/core/client/server/msgbus_server_pq_read_session_info.h>
-
-#include <kikimr/yndx/grpc_services/persqueue/grpc_pq_actor.h>
-
-
-namespace NKikimr {
-namespace NMsgBusProxy {
-
-class TPersQueueGetReadSessionsInfoWorkerWithPQv0 : public IPersQueueGetReadSessionsInfoWorker {
-public:
- using TBase = IPersQueueGetReadSessionsInfoWorker;
- using TBase::TBase;
- using TBase::SendStatusRequest;
-
- STFUNC(StateFunc) override {
- switch (ev->GetTypeRewrite()) {
- HFunc(NGRpcProxy::TEvPQProxy::TEvReadSessionStatusResponse, HandleStatusResponse<NGRpcProxy::TEvPQProxy::TEvReadSessionStatusResponse>);
- HFunc(NGRpcProxy::V1::TEvPQProxy::TEvReadSessionStatusResponse, HandleStatusResponse<NGRpcProxy::V1::TEvPQProxy::TEvReadSessionStatusResponse>);
- HFunc(TEvents::TEvUndelivered, Undelivered);
- HFunc(TEvInterconnect::TEvNodeDisconnected, Disconnected);
- }
- }
-
-private:
- void SendStatusRequest(const TString& sessionName, TActorId actorId, const TActorContext& ctx) override;
-};
-
-class TPersQueueGetReadSessionsInfoWorkerWithPQv0Factory : public IPersQueueGetReadSessionsInfoWorkerFactory {
-public:
- THolder<IPersQueueGetReadSessionsInfoWorker> Create(
- const TActorId& parentId,
- const THashMap<TString, TActorId>& readSessions,
- std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo
- ) const override {
- return MakeHolder<TPersQueueGetReadSessionsInfoWorkerWithPQv0>(parentId, readSessions, nodesInfo);
- }
-};
-
-} // namespace NMsgBusProxy
-} // namespace NKikimr
-
diff --git a/kikimr/yndx/persqueue/read_batch_converter/CMakeLists.txt b/kikimr/yndx/persqueue/read_batch_converter/CMakeLists.txt
deleted file mode 100644
index 9e8f9b6006..0000000000
--- a/kikimr/yndx/persqueue/read_batch_converter/CMakeLists.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(yndx-persqueue-read_batch_converter)
-target_link_libraries(yndx-persqueue-read_batch_converter PUBLIC
- contrib-libs-cxxsupp
- yutil
- api-protos-yndx
-)
-target_sources(yndx-persqueue-read_batch_converter PRIVATE
- ${CMAKE_SOURCE_DIR}/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.cpp
-)
diff --git a/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.cpp b/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.cpp
deleted file mode 100644
index bca03dc72f..0000000000
--- a/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-#include "read_batch_converter.h"
-
-namespace NPersQueue {
-
-static void Convert(const ReadResponse::BatchedData::PartitionData& partition, ReadResponse::Data::MessageBatch* dstBatch) {
- dstBatch->set_topic(partition.topic());
- dstBatch->set_partition(partition.partition());
- for (const ReadResponse::BatchedData::Batch& batch : partition.batch()) {
- for (const ReadResponse::BatchedData::MessageData& message : batch.message_data()) {
- ReadResponse::Data::Message* const dstMessage = dstBatch->add_message();
- dstMessage->set_data(message.data());
- dstMessage->set_offset(message.offset());
-
- MessageMeta* const meta = dstMessage->mutable_meta();
- meta->set_source_id(batch.source_id());
- meta->set_seq_no(message.seq_no());
- meta->set_create_time_ms(message.create_time_ms());
- meta->set_write_time_ms(batch.write_time_ms());
- meta->set_codec(message.codec());
- meta->set_ip(batch.ip());
- meta->set_uncompressed_size(message.uncompressed_size());
- if (batch.has_extra_fields()) {
- *meta->mutable_extra_fields() = batch.extra_fields();
- }
- }
- }
-}
-
-void ConvertToOldBatch(ReadResponse& response) {
- if (!response.has_batched_data()) {
- return;
- }
- ReadResponse::BatchedData data;
- data.Swap(response.mutable_batched_data());
-
- ReadResponse::Data& dstData = *response.mutable_data(); // this call will clear BatchedData field
- dstData.set_cookie(data.cookie());
- for (const ReadResponse::BatchedData::PartitionData& partition : data.partition_data()) {
- Convert(partition, dstData.add_message_batch());
- }
-}
-
-} // namespace NPersQueue
diff --git a/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h b/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h
deleted file mode 100644
index ddf32da4ae..0000000000
--- a/kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#pragma once
-#include <kikimr/yndx/api/protos/persqueue.pb.h>
-
-namespace NPersQueue {
-
-// Converts responses with BatchedData field to responses with Data field.
-// Other responses will be leaved unchanged.
-void ConvertToOldBatch(ReadResponse& response);
-
-} // namespace NPersQueue
diff --git a/library/cpp/grpc/common/CMakeLists.txt b/library/cpp/grpc/common/CMakeLists.txt
deleted file mode 100644
index ea5aa6bf3d..0000000000
--- a/library/cpp/grpc/common/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-grpc-common INTERFACE)
-target_link_libraries(cpp-grpc-common INTERFACE
- contrib-libs-cxxsupp
- yutil
- contrib-libs-grpc
-)
diff --git a/library/cpp/grpc/common/time_point.h b/library/cpp/grpc/common/time_point.h
deleted file mode 100644
index c2b8126297..0000000000
--- a/library/cpp/grpc/common/time_point.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-#include <contrib/libs/grpc/include/grpcpp/support/time.h>
-
-#include <util/datetime/base.h>
-
-#include <chrono>
-
-namespace grpc {
-// Specialization of TimePoint for TInstant
-template <>
-class TimePoint<TInstant> : public TimePoint<std::chrono::system_clock::time_point> {
- using TChronoDuration = std::chrono::duration<TDuration::TValue, std::micro>;
-
-public:
- TimePoint(const TInstant& time)
- : TimePoint<std::chrono::system_clock::time_point>(
- std::chrono::system_clock::time_point(
- std::chrono::duration_cast<std::chrono::system_clock::duration>(
- TChronoDuration(time.GetValue())))) {
- }
-};
-} // namespace grpc
diff --git a/library/cpp/http/simple/.arcignore b/library/cpp/http/simple/.arcignore
deleted file mode 100644
index 2674132024..0000000000
--- a/library/cpp/http/simple/.arcignore
+++ /dev/null
@@ -1 +0,0 @@
-*-ut*
diff --git a/library/cpp/http/simple/CMakeLists.txt b/library/cpp/http/simple/CMakeLists.txt
deleted file mode 100644
index 317a748a89..0000000000
--- a/library/cpp/http/simple/CMakeLists.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-http-simple)
-target_link_libraries(cpp-http-simple PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-http-io
- cpp-openssl-io
- cpp-string_utils-url
- library-cpp-uri
-)
-target_sources(cpp-http-simple PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/http/simple/http_client.cpp
-)
diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp
deleted file mode 100644
index 818dc048ad..0000000000
--- a/library/cpp/http/simple/http_client.cpp
+++ /dev/null
@@ -1,356 +0,0 @@
-#include "http_client.h"
-
-#include <library/cpp/string_utils/url/url.h>
-#include <library/cpp/uri/http_url.h>
-
-#include <util/stream/output.h>
-#include <util/string/cast.h>
-#include <util/string/join.h>
-#include <util/string/split.h>
-
-TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host,
- ui32 port,
- TDuration socketTimeout,
- TDuration connectTimeout)
- : Host(CutHttpPrefix(host))
- , Port(port)
- , SocketTimeout(socketTimeout)
- , ConnectTimeout(connectTimeout)
- , IsHttps(host.StartsWith("https"))
- , IsClosingRequired(false)
- , HttpsVerification(TVerifyCert{Host})
- , IfResponseRequired([](const THttpInput&) { return true; })
-{
-}
-
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoGet(const TStringBuf relativeUrl,
- IOutputStream* output,
- const THeaders& headers,
- THttpHeaders* outHeaders) {
- return DoRequest(TStringBuf("GET"),
- relativeUrl,
- {},
- output,
- headers,
- outHeaders);
-}
-
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoPost(const TStringBuf relativeUrl,
- const TStringBuf body,
- IOutputStream* output,
- const THeaders& headers,
- THttpHeaders* outHeaders) {
- return DoRequest(TStringBuf("POST"),
- relativeUrl,
- body,
- output,
- headers,
- outHeaders);
-}
-
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequest(const TStringBuf method,
- const TStringBuf relativeUrl,
- const TStringBuf body,
- IOutputStream* output,
- const THeaders& inHeaders,
- THttpHeaders* outHeaders) {
- const TString contentLength = IntToString<10, size_t>(body.size());
- return DoRequestReliable(FormRequest(method, relativeUrl, body, inHeaders, contentLength), output, outHeaders);
-}
-
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestRaw(const TStringBuf raw,
- IOutputStream* output,
- THttpHeaders* outHeaders) {
- return DoRequestReliable(raw, output, outHeaders);
-}
-
-void TKeepAliveHttpClient::DisableVerificationForHttps() {
- HttpsVerification.Clear();
- Connection.Reset();
-}
-
-void TKeepAliveHttpClient::SetClientCertificate(const TOpenSslClientIO::TOptions::TClientCert& options) {
- ClientCertificate = options;
-}
-
-void TKeepAliveHttpClient::ResetConnection() {
- Connection.Reset();
-}
-
-TVector<IOutputStream::TPart> TKeepAliveHttpClient::FormRequest(TStringBuf method,
- const TStringBuf relativeUrl,
- TStringBuf body,
- const TKeepAliveHttpClient::THeaders& headers,
- TStringBuf contentLength) const {
- TVector<IOutputStream::TPart> parts;
-
- parts.reserve(16 + 4 * headers.size());
- parts.push_back(method);
- parts.push_back(TStringBuf(" "));
- parts.push_back(relativeUrl);
- parts.push_back(TStringBuf(" HTTP/1.1"));
- parts.push_back(IOutputStream::TPart::CrLf());
- parts.push_back(TStringBuf("Host: "));
- parts.push_back(TStringBuf(Host));
- parts.push_back(IOutputStream::TPart::CrLf());
- parts.push_back(TStringBuf("Content-Length: "));
- parts.push_back(contentLength);
- parts.push_back(IOutputStream::TPart::CrLf());
-
- for (const auto& entry : headers) {
- parts.push_back(IOutputStream::TPart(entry.first));
- parts.push_back(IOutputStream::TPart(TStringBuf(": ")));
- parts.push_back(IOutputStream::TPart(entry.second));
- parts.push_back(IOutputStream::TPart::CrLf());
- }
-
- parts.push_back(IOutputStream::TPart::CrLf());
- if (body) {
- parts.push_back(IOutputStream::TPart(body));
- }
-
- return parts;
-}
-
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::ReadAndTransferHttp(THttpInput& input,
- IOutputStream* output,
- THttpHeaders* outHeaders) const {
- TKeepAliveHttpClient::THttpCode statusCode;
- try {
- statusCode = ParseHttpRetCode(input.FirstLine());
- } catch (TFromStringException& e) {
- TString rest = input.ReadAll();
- ythrow THttpRequestException() << "Failed parse status code in response of " << Host << ": " << e.what() << " (" << input.FirstLine() << ")"
- << "\nFull http response:\n"
- << rest;
- }
-
- auto canContainBody = [](auto statusCode) {
- return statusCode != HTTP_NOT_MODIFIED && statusCode != HTTP_NO_CONTENT;
- };
-
- if (output && canContainBody(statusCode) && IfResponseRequired(input)) {
- TransferData(&input, output);
- }
- if (outHeaders) {
- *outHeaders = input.Headers();
- }
-
- return statusCode;
-}
-
-THttpInput* TKeepAliveHttpClient::GetHttpInput() {
- return Connection ? Connection->GetHttpInput() : nullptr;
-}
-
-bool TKeepAliveHttpClient::CreateNewConnectionIfNeeded() {
- if (IsClosingRequired || (Connection && !Connection->IsOk())) {
- Connection.Reset();
- }
- if (!Connection) {
- Connection = MakeHolder<NPrivate::THttpConnection>(Host,
- Port,
- SocketTimeout,
- ConnectTimeout,
- IsHttps,
- ClientCertificate,
- HttpsVerification);
- IsClosingRequired = false;
- return true;
- }
- return false;
-}
-
-THttpRequestException::THttpRequestException(int statusCode)
- : StatusCode(statusCode)
-{
-}
-
-int THttpRequestException::GetStatusCode() const {
- return StatusCode;
-}
-
-TSimpleHttpClient::TSimpleHttpClient(const TOptions& options)
- : Host(options.Host())
- , Port(options.Port())
- , SocketTimeout(options.SocketTimeout())
- , ConnectTimeout(options.ConnectTimeout())
-{
-}
-
-TSimpleHttpClient::TSimpleHttpClient(const TString& host, ui32 port, TDuration socketTimeout, TDuration connectTimeout)
- : Host(host)
- , Port(port)
- , SocketTimeout(socketTimeout)
- , ConnectTimeout(connectTimeout)
-{
-}
-
-void TSimpleHttpClient::EnableVerificationForHttps() {
- HttpsVerification = true;
-}
-
-void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers) const {
- TKeepAliveHttpClient cl = CreateClient();
-
- TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers);
-
- Y_ENSURE(cl.GetHttpInput());
- ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
-}
-
-void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers) const {
- TKeepAliveHttpClient cl = CreateClient();
-
- TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers);
-
- Y_ENSURE(cl.GetHttpInput());
- ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
-}
-
-void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output) const {
- TKeepAliveHttpClient cl = CreateClient();
-
- TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output);
-
- Y_ENSURE(cl.GetHttpInput());
- ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
-}
-
-namespace NPrivate {
- THttpConnection::THttpConnection(const TString& host,
- ui32 port,
- TDuration sockTimeout,
- TDuration connTimeout,
- bool isHttps,
- const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert,
- const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert)
- : Addr(Resolve(host, port))
- , Socket(Connect(Addr, sockTimeout, connTimeout, host, port))
- , SocketIn(Socket)
- , SocketOut(Socket)
- {
- if (isHttps) {
- TOpenSslClientIO::TOptions opts;
- if (clientCert) {
- opts.ClientCert_ = clientCert;
- }
- if (verifyCert) {
- opts.VerifyCert_ = verifyCert;
- }
-
- Ssl = MakeHolder<TOpenSslClientIO>(&SocketIn, &SocketOut, opts);
- HttpOut = MakeHolder<THttpOutput>(Ssl.Get());
- } else {
- HttpOut = MakeHolder<THttpOutput>(&SocketOut);
- }
-
- HttpOut->EnableKeepAlive(true);
- }
-
- TNetworkAddress THttpConnection::Resolve(const TString& host, ui32 port) {
- try {
- return TNetworkAddress(host, port);
- } catch (const yexception& e) {
- ythrow THttpRequestException() << "Resolve of " << host << ": " << e.what();
- }
- }
-
- TSocket THttpConnection::Connect(TNetworkAddress& addr,
- TDuration sockTimeout,
- TDuration connTimeout,
- const TString& host,
- ui32 port) {
- try {
- TSocket socket(addr, connTimeout);
- TDuration socketTimeout = Max(sockTimeout, TDuration::MilliSeconds(1)); // timeout less than 1ms will be interpreted as 0 in SetSocketTimeout() call below and will result in infinite wait
-
- ui32 seconds = socketTimeout.Seconds();
- ui32 milliSeconds = (socketTimeout - TDuration::Seconds(seconds)).MilliSeconds();
- socket.SetSocketTimeout(seconds, milliSeconds);
- return socket;
- } catch (const yexception& e) {
- ythrow THttpRequestException() << "Connect to " << host << ':' << port << " failed: " << e.what();
- }
- }
-}
-
-void TSimpleHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream*, const unsigned statusCode) const {
- if (!(statusCode >= 200 && statusCode < 300)) {
- TString rest = input.ReadAll();
- ythrow THttpRequestException(statusCode) << "Got " << statusCode << " at " << Host << relativeUrl << "\nFull http response:\n"
- << rest;
- }
-}
-
-TSimpleHttpClient::~TSimpleHttpClient() {
-}
-
-TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const {
- TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout);
-
- if (!HttpsVerification) {
- cl.DisableVerificationForHttps();
- }
-
- PrepareClient(cl);
-
- return cl;
-}
-
-void TSimpleHttpClient::PrepareClient(TKeepAliveHttpClient&) const {
-}
-
-TRedirectableHttpClient::TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout, TDuration connectTimeout)
- : TSimpleHttpClient(host, port, socketTimeout, connectTimeout)
-{
-}
-
-void TRedirectableHttpClient::PrepareClient(TKeepAliveHttpClient& cl) const {
- cl.IfResponseRequired = [](const THttpInput& input) {
- return !input.Headers().HasHeader("Location");
- };
-}
-
-void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const {
- for (auto i = input.Headers().Begin(), e = input.Headers().End(); i != e; ++i) {
- if (0 == TString::compare(i->Name(), TStringBuf("Location"))) {
- TVector<TString> request_url_parts, request_body_parts;
-
- size_t splitted_index = 0;
- for (auto& iter : StringSplitter(i->Value()).Split('/')) {
- if (splitted_index < 3) {
- request_url_parts.push_back(TString(iter.Token()));
- } else {
- request_body_parts.push_back(TString(iter.Token()));
- }
- ++splitted_index;
- }
-
- TString url = JoinSeq("/", request_url_parts);
- ui16 port = 443;
-
- THttpURL u;
- if (THttpURL::ParsedOK == u.Parse(url)) {
- const char* p = u.Get(THttpURL::FieldPort);
- if (p) {
- port = FromString<ui16>(p);
- url = u.PrintS(THttpURL::FlagScheme | THttpURL::FlagHost);
- }
- }
-
- TRedirectableHttpClient cl(url, port, TDuration::Seconds(60), TDuration::Seconds(60));
- if (HttpsVerification) {
- cl.EnableVerificationForHttps();
- }
- cl.DoGet(TString("/") + JoinSeq("/", request_body_parts), output);
- return;
- }
- }
- if (!(statusCode >= 200 && statusCode < 300)) {
- TString rest = input.ReadAll();
- ythrow THttpRequestException(statusCode) << "Got " << statusCode << " at " << Host << relativeUrl << "\nFull http response:\n"
- << rest;
- }
- TransferData(&input, output);
-}
diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h
deleted file mode 100644
index 94ee487202..0000000000
--- a/library/cpp/http/simple/http_client.h
+++ /dev/null
@@ -1,276 +0,0 @@
-#pragma once
-
-#include "http_client_options.h"
-
-#include <util/datetime/base.h>
-#include <util/generic/hash.h>
-#include <util/generic/ptr.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/yexception.h>
-#include <util/network/socket.h>
-
-#include <library/cpp/http/io/stream.h>
-#include <library/cpp/http/misc/httpcodes.h>
-#include <library/cpp/openssl/io/stream.h>
-
-class TNetworkAddress;
-class IOutputStream;
-class TSocket;
-
-namespace NPrivate {
- class THttpConnection;
-}
-
-/*!
- * HTTPS is supported in two modes.
- * HTTPS verification enabled by default in TKeepAliveHttpClient and disabled by default in TSimpleHttpClient.
- * HTTPS verification requires valid private certificate on server side and valid public certificate on client side.
- *
- * For client:
- * Uses builtin certs.
- * Also uses default CA path /etc/ssl/certs/ - can be provided with debian package: ca-certificates.deb.
- * It can be expanded with ENV: SSL_CERT_DIR.
- */
-
-/*!
- * TKeepAliveHttpClient can keep connection alive with HTTP and HTTPS only if you use the same instance of class.
- * It closes connection on every socket/network error and throws error.
- * For example, HTTP code == 500 is NOT error - connection will be still open.
- * It is THREAD UNSAFE because it stores connection state in attributes.
- * If you need thread safe client, look at TSimpleHttpClient
- */
-
-class TKeepAliveHttpClient {
-public:
- using THeaders = THashMap<TString, TString>;
- using THttpCode = unsigned;
-
-public:
- TKeepAliveHttpClient(const TString& host,
- ui32 port,
- TDuration socketTimeout = TDuration::Seconds(5),
- TDuration connectTimeout = TDuration::Seconds(30));
-
- THttpCode DoGet(const TStringBuf relativeUrl,
- IOutputStream* output = nullptr,
- const THeaders& headers = THeaders(),
- THttpHeaders* outHeaders = nullptr);
-
- // builds post request from headers and body
- THttpCode DoPost(const TStringBuf relativeUrl,
- const TStringBuf body,
- IOutputStream* output = nullptr,
- const THeaders& headers = THeaders(),
- THttpHeaders* outHeaders = nullptr);
-
- // builds request with any HTTP method from headers and body
- THttpCode DoRequest(const TStringBuf method,
- const TStringBuf relativeUrl,
- const TStringBuf body,
- IOutputStream* output = nullptr,
- const THeaders& inHeaders = THeaders(),
- THttpHeaders* outHeaders = nullptr);
-
- // requires already well-formed request
- THttpCode DoRequestRaw(const TStringBuf raw,
- IOutputStream* output = nullptr,
- THttpHeaders* outHeaders = nullptr);
-
- void DisableVerificationForHttps();
- void SetClientCertificate(const TOpenSslClientIO::TOptions::TClientCert& options);
-
- void ResetConnection();
-
- const TString& GetHost() const {
- return Host;
- }
-
- ui32 GetPort() const {
- return Port;
- }
-
-private:
- template <class T>
- THttpCode DoRequestReliable(const T& raw,
- IOutputStream* output,
- THttpHeaders* outHeaders);
-
- TVector<IOutputStream::TPart> FormRequest(TStringBuf method, const TStringBuf relativeUrl,
- TStringBuf body,
- const THeaders& headers, TStringBuf contentLength) const;
-
- THttpCode ReadAndTransferHttp(THttpInput& input, IOutputStream* output, THttpHeaders* outHeaders) const;
-
- bool CreateNewConnectionIfNeeded(); // Returns true if now we have a new connection.
-
-private:
- using TVerifyCert = TOpenSslClientIO::TOptions::TVerifyCert;
- using TClientCert = TOpenSslClientIO::TOptions::TClientCert;
-
- const TString Host;
- const ui32 Port;
- const TDuration SocketTimeout;
- const TDuration ConnectTimeout;
- const bool IsHttps;
-
- THolder<NPrivate::THttpConnection> Connection;
- bool IsClosingRequired;
- TMaybe<TClientCert> ClientCertificate;
- TMaybe<TVerifyCert> HttpsVerification;
-
-private:
- THttpInput* GetHttpInput();
-
- using TIfResponseRequired = std::function<bool(const THttpInput&)>;
- TIfResponseRequired IfResponseRequired;
-
- friend class TSimpleHttpClient;
- friend class TRedirectableHttpClient;
-};
-
-class THttpRequestException: public yexception {
-private:
- int StatusCode;
-
-public:
- THttpRequestException(int statusCode = 0);
- int GetStatusCode() const;
-};
-
-/*!
- * TSimpleHttpClient can NOT keep connection alive.
- * It closes connection after each request.
- * HTTP code < 200 || code >= 300 is error - exception will be thrown.
- * It is THREAD SAFE because it stores only consts.
- */
-
-class TSimpleHttpClient {
-protected:
- using TVerifyCert = TKeepAliveHttpClient::TVerifyCert;
-
- const TString Host;
- const ui32 Port;
- const TDuration SocketTimeout;
- const TDuration ConnectTimeout;
- bool HttpsVerification = false;
-
-public:
- using THeaders = TKeepAliveHttpClient::THeaders;
- using TOptions = TSimpleHttpClientOptions;
-
-public:
- explicit TSimpleHttpClient(const TOptions& options);
-
- TSimpleHttpClient(const TString& host, ui32 port,
- TDuration socketTimeout = TDuration::Seconds(5), TDuration connectTimeout = TDuration::Seconds(30));
-
- void EnableVerificationForHttps();
-
- void DoGet(const TStringBuf relativeUrl, IOutputStream* output, const THeaders& headers = THeaders()) const;
-
- // builds post request from headers and body
- void DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THeaders& headers = THeaders()) const;
-
- // requires already well-formed post request
- void DoPostRaw(const TStringBuf relativeUrl, TStringBuf rawRequest, IOutputStream* output) const;
-
- virtual ~TSimpleHttpClient();
-
-private:
- TKeepAliveHttpClient CreateClient() const;
-
- virtual void PrepareClient(TKeepAliveHttpClient& cl) const;
- virtual void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const;
-};
-
-class TRedirectableHttpClient: public TSimpleHttpClient {
-public:
- TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout = TDuration::Seconds(5),
- TDuration connectTimeout = TDuration::Seconds(30));
-
-private:
- void PrepareClient(TKeepAliveHttpClient& cl) const override;
- void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const override;
-};
-
-namespace NPrivate {
- class THttpConnection {
- public:
- THttpConnection(const TString& host,
- ui32 port,
- TDuration sockTimeout,
- TDuration connTimeout,
- bool isHttps,
- const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert,
- const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert);
-
- bool IsOk() const {
- return IsNotSocketClosedByOtherSide(Socket);
- }
-
- template <typename TContainer>
- void Write(const TContainer& request) {
- HttpOut->Write(request.data(), request.size());
- HttpIn = Ssl ? MakeHolder<THttpInput>(Ssl.Get())
- : MakeHolder<THttpInput>(&SocketIn);
- HttpOut->Flush();
- }
-
- THttpInput* GetHttpInput() {
- return HttpIn.Get();
- }
-
- private:
- static TNetworkAddress Resolve(const TString& host, ui32 port);
-
- static TSocket Connect(TNetworkAddress& addr,
- TDuration sockTimeout,
- TDuration connTimeout,
- const TString& host,
- ui32 port);
-
- private:
- TNetworkAddress Addr;
- TSocket Socket;
- TSocketInput SocketIn;
- TSocketOutput SocketOut;
- THolder<TOpenSslClientIO> Ssl;
- THolder<THttpInput> HttpIn;
- THolder<THttpOutput> HttpOut;
- };
-}
-
-template <class T>
-TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T& raw,
- IOutputStream* output,
- THttpHeaders* outHeaders) {
- for (int i = 0; i < 2; ++i) {
- const bool haveNewConnection = CreateNewConnectionIfNeeded();
- const bool couldRetry = !haveNewConnection && i == 0; // Actually old connection could be already closed by server,
- // so we should try one more time in this case.
- try {
- Connection->Write(raw);
-
- THttpCode code = ReadAndTransferHttp(*Connection->GetHttpInput(), output, outHeaders);
- if (!Connection->GetHttpInput()->IsKeepAlive()) {
- IsClosingRequired = true;
- }
- return code;
- } catch (const TSystemError& e) {
- Connection.Reset();
- if (!couldRetry || e.Status() != EPIPE) {
- throw;
- }
- } catch (const THttpReadException&) { // Actually old connection is already closed by server
- Connection.Reset();
- if (!couldRetry) {
- throw;
- }
- } catch (const std::exception&) {
- Connection.Reset();
- throw;
- }
- }
- Y_FAIL(); // We should never be here.
- return 0;
-}
diff --git a/library/cpp/http/simple/http_client_options.h b/library/cpp/http/simple/http_client_options.h
deleted file mode 100644
index f2e964a462..0000000000
--- a/library/cpp/http/simple/http_client_options.h
+++ /dev/null
@@ -1,59 +0,0 @@
-#pragma once
-
-#include <util/datetime/base.h>
-#include <library/cpp/string_utils/url/url.h>
-
-class TSimpleHttpClientOptions {
- using TSelf = TSimpleHttpClientOptions;
-
-public:
- TSimpleHttpClientOptions() = default;
-
- explicit TSimpleHttpClientOptions(TStringBuf url) {
- TStringBuf scheme, host;
- GetSchemeHostAndPort(url, scheme, host, Port_);
- Host_ = url.Head(scheme.size() + host.size());
- }
-
- TSelf& Host(TStringBuf host) {
- Host_ = host;
- return *this;
- }
-
- const TString& Host() const noexcept {
- return Host_;
- }
-
- TSelf& Port(ui16 port) {
- Port_ = port;
- return *this;
- }
-
- ui16 Port() const noexcept {
- return Port_;
- }
-
- TSelf& SocketTimeout(TDuration timeout) {
- SocketTimeout_ = timeout;
- return *this;
- }
-
- TDuration SocketTimeout() const noexcept {
- return SocketTimeout_;
- }
-
- TSelf& ConnectTimeout(TDuration timeout) {
- ConnectTimeout_ = timeout;
- return *this;
- }
-
- TDuration ConnectTimeout() const noexcept {
- return ConnectTimeout_;
- }
-
-private:
- TString Host_;
- ui16 Port_;
- TDuration SocketTimeout_ = TDuration::Seconds(5);
- TDuration ConnectTimeout_ = TDuration::Seconds(30);
-};
diff --git a/library/cpp/http/simple/ut/http_ut.cpp b/library/cpp/http/simple/ut/http_ut.cpp
deleted file mode 100644
index bf7e767428..0000000000
--- a/library/cpp/http/simple/ut/http_ut.cpp
+++ /dev/null
@@ -1,439 +0,0 @@
-#include <library/cpp/http/simple/http_client.h>
-
-#include <library/cpp/http/server/response.h>
-
-#include <library/cpp/testing/mock_server/server.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/system/event.h>
-#include <util/system/thread.h>
-
-#include <thread>
-
-Y_UNIT_TEST_SUITE(SimpleHttp) {
- static THttpServerOptions createOptions(ui16 port, bool keepAlive) {
- THttpServerOptions o;
- o.AddBindAddress("localhost", port);
- o.SetThreads(1);
- o.SetMaxConnections(1);
- o.SetMaxQueueSize(1);
- o.EnableKeepAlive(keepAlive);
- return o;
- }
-
- class TPong: public TRequestReplier {
- TDuration Sleep_;
- ui16 Port_;
-
- public:
- TPong(TDuration sleep = TDuration(), ui16 port = 80)
- : Sleep_(sleep)
- , Port_(port)
- {
- }
-
- bool DoReply(const TReplyParams& params) override {
- TStringBuf path = TParsedHttpFull(params.Input.FirstLine()).Path;
- params.Input.ReadAll();
- if (path == "/redirect") {
- params.Output << "HTTP/1.1 307 Internal Redirect\r\n"
- "Location: http://localhost:"
- << Port_
- << "/redirect2?some_param=qwe\r\n"
- "Non-Authoritative-Reason: HSTS\r\n\r\n"
- "must be missing";
- return true;
- }
-
- if (path == "/redirect2") {
- UNIT_ASSERT_VALUES_EQUAL("some_param=qwe", TParsedHttpFull(params.Input.FirstLine()).Cgi);
- params.Output << "HTTP/1.1 307 Internal Redirect\r\n"
- "Location: http://localhost:"
- << Port_
- << "/ping\r\n"
- "Non-Authoritative-Reason: HSTS\r\n\r\n"
- "must be missing too";
- return true;
- }
-
- if (path != "/ping") {
- UNIT_ASSERT_C(false, "path is incorrect: '" << path << "'");
- }
-
- Sleep(Sleep_);
-
- THttpResponse resp(HTTP_OK);
- resp.SetContent("pong");
- resp.OutTo(params.Output);
-
- return true;
- }
- };
-
- class TCodedPong: public TRequestReplier {
- HttpCodes Code_;
-
- public:
- TCodedPong(HttpCodes code)
- : Code_(code)
- {
- }
-
- bool DoReply(const TReplyParams& params) override {
- if (TParsedHttpFull(params.Input.FirstLine()).Path != "/ping") {
- UNIT_ASSERT(false);
- }
-
- THttpResponse resp(Code_);
- resp.SetContent("pong");
- resp.OutTo(params.Output);
-
- return true;
- }
- };
-
- class T500: public TRequestReplier {
- ui16 Port_;
-
- public:
- T500(ui16 port)
- : Port_(port)
- {
- }
-
- bool DoReply(const TReplyParams& params) override {
- TStringBuf path = TParsedHttpFull(params.Input.FirstLine()).Path;
-
- if (path == "/bad_redirect") {
- params.Output << "HTTP/1.1 500 Internal Redirect\r\n"
- "Location: http://localhost:1/qwerty\r\n"
- "Non-Authoritative-Reason: HSTS\r\n\r\n";
- return true;
- }
-
- if (path == "/redirect_to_500") {
- params.Output << "HTTP/1.1 307 Internal Redirect\r\n"
- "Location: http://localhost:"
- << Port_
- << "/500\r\n"
- "Non-Authoritative-Reason: HSTS\r\n\r\n";
- return true;
- }
-
- THttpResponse resp(HTTP_INTERNAL_SERVER_ERROR);
- resp.SetContent("bang");
- resp.OutTo(params.Output);
-
- return true;
- }
- };
-
- Y_UNIT_TEST(simpleSuccessful) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, false), []() { return new TPong; });
-
- TSimpleHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoPost("/ping", "", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoPost("/ping", "", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- }
-
- Y_UNIT_TEST(simpleMessages) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, false), []() { return new TPong; });
-
- TSimpleHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- {
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", nullptr));
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- server.SetGenerator([]() { return new TCodedPong(HTTP_CONTINUE); });
- {
- TStringStream s;
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoPost("/ping", "", &s),
- THttpRequestException,
- "Got 100 at localhost/ping\n"
- "Full http response:\n");
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- {
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoPost("/ping", "", nullptr),
- THttpRequestException,
- "Got 100 at localhost/ping\n"
- "Full http response:\n"
- "pong");
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- }
-
- Y_UNIT_TEST(simpleTimeout) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), []() { return new TPong(TDuration::MilliSeconds(300)); });
-
- TSimpleHttpClient cl("localhost", port, TDuration::MilliSeconds(50), TDuration::MilliSeconds(50));
-
- TStringStream s;
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/ping", &s),
- TSystemError,
- "Resource temporarily unavailable");
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoPost("/ping", "", &s),
- TSystemError,
- "Resource temporarily unavailable");
- }
-
- Y_UNIT_TEST(simpleError) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), []() { return new TPong; });
-
- TSimpleHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- {
- TStringStream s;
- server.SetGenerator([]() { return new TCodedPong(HTTP_CONTINUE); });
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/ping", &s),
- THttpRequestException,
- "Got 100 at localhost/ping\n"
- "Full http response:");
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- {
- TStringStream s;
- server.SetGenerator([]() { return new TCodedPong(HTTP_OK); });
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- server.SetGenerator([]() { return new TCodedPong(HTTP_PARTIAL_CONTENT); });
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping", &s));
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- {
- TStringStream s;
- server.SetGenerator([]() { return new TCodedPong(HTTP_MULTIPLE_CHOICES); });
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/ping", &s),
- THttpRequestException,
- "Got 300 at localhost/ping\n"
- "Full http response:");
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
- }
-
- Y_UNIT_TEST(redirectable) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), [port]() { return new TPong(TDuration(), port); });
-
- TRedirectableHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- {
- TStringStream s;
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/redirect", &s));
- UNIT_ASSERT_VALUES_EQUAL("pong", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- server.SetGenerator([port]() { return new T500(port); });
-
- TStringStream s;
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/bad_redirect", &s),
- THttpRequestException,
- "can not connect to ");
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/redirect_to_500", &s),
- THttpRequestException,
- "Got 500 at http://localhost/500\n"
- "Full http response:\n");
- UNIT_ASSERT_VALUES_EQUAL("bang", s.Str());
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- }
-
- Y_UNIT_TEST(keepaliveSuccessful) {
- auto test = [](bool keepalive, i64 clientCount) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, keepalive), []() { return new TPong; });
-
- TKeepAliveHttpClient cl("localhost", port);
- UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
- {
- TStringStream s;
- int code = -1;
- UNIT_ASSERT_NO_EXCEPTION_C(code = cl.DoGet("/ping", &s), keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C(200, code, keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C("pong", s.Str(), keepalive);
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(clientCount, server.GetClientCount());
- }
- {
- TStringStream s;
- int code = -1;
- UNIT_ASSERT_NO_EXCEPTION_C(code = cl.DoGet("/ping", &s), keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C(200, code, keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C("pong", s.Str(), keepalive);
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(clientCount, server.GetClientCount());
- }
-
- {
- TStringStream s;
- int code = -1;
- UNIT_ASSERT_NO_EXCEPTION_C(code = cl.DoPost("/ping", "", &s), keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C(200, code, keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C("pong", s.Str(), keepalive);
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(clientCount, server.GetClientCount());
- }
- {
- TStringStream s;
- int code = -1;
- UNIT_ASSERT_NO_EXCEPTION_C(code = cl.DoPost("/ping", "", &s), keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C(200, code, keepalive);
- UNIT_ASSERT_VALUES_EQUAL_C("pong", s.Str(), keepalive);
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(clientCount, server.GetClientCount());
- }
- };
-
- test(true, 1);
- test(false, 0);
- }
-
- Y_UNIT_TEST(keepaliveTimeout) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), []() { return new TPong(TDuration::MilliSeconds(300)); });
-
- TKeepAliveHttpClient cl("localhost", port, TDuration::MilliSeconds(50), TDuration::MilliSeconds(50));
-
- TStringStream s;
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/ping", &s),
- TSystemError,
- "Resource temporarily unavailable");
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoPost("/ping", "", &s),
- TSystemError,
- "Resource temporarily unavailable");
- }
-
- Y_UNIT_TEST(keepaliveHeaders) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), []() { return new TPong; });
-
- TKeepAliveHttpClient cl("localhost", port);
-
- TStringStream s;
- THttpHeaders h;
- UNIT_ASSERT_VALUES_EQUAL(200, cl.DoGet("/ping", &s, {}, &h));
- TStringStream hs;
- h.OutTo(&hs);
- UNIT_ASSERT_VALUES_EQUAL("Content-Length: 4\r\nConnection: Keep-Alive\r\n", hs.Str());
- }
-
- Y_UNIT_TEST(keepaliveRaw) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(createOptions(port, true), []() { return new TPong; });
-
- TKeepAliveHttpClient cl("localhost", port);
-
- TStringStream s;
- THttpHeaders h;
-
- TString raw = "POST /ping HTTP/1.1\r\n"
- "Connection: Keep-Alive\r\n"
- "Accept-Encoding: gzip, deflate\r\n"
- "Content-Length: 9\r\n"
- "Content-Type: application/x-www-form-urlencoded\r\n"
- "User-Agent: Python-urllib/2.6\r\n"
- "\r\n"
- "some body";
-
- UNIT_ASSERT_VALUES_EQUAL(200, cl.DoRequestRaw(raw, &s, &h));
- TStringStream hs;
- h.OutTo(&hs);
- UNIT_ASSERT_VALUES_EQUAL("Content-Length: 4\r\nConnection: Keep-Alive\r\n", hs.Str());
-
- raw = "GET /ping HT TP/1.1\r\n";
- UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoRequestRaw(raw, &s, &h), TSystemError, "can not read from socket input stream");
- }
-
- Y_UNIT_TEST(keepaliveWithClosedByPeer) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer::TGenerator gen = []() { return new TPong; };
- THolder<NMock::TMockServer> server = MakeHolder<NMock::TMockServer>(createOptions(port, true), gen);
-
- TKeepAliveHttpClient cl("localhost", port);
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping"));
-
- server.Reset();
- server = MakeHolder<NMock::TMockServer>(createOptions(port, true), gen);
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping"));
-
- TKeepAliveHttpClient cl2("localhost", port);
- UNIT_ASSERT_NO_EXCEPTION(cl2.DoGet("/ping"));
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/ping"));
- }
-}
diff --git a/library/cpp/http/simple/ut/https_server/http_server.crt b/library/cpp/http/simple/ut/https_server/http_server.crt
deleted file mode 100644
index 74d74fafea..0000000000
--- a/library/cpp/http/simple/ut/https_server/http_server.crt
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDATCCAemgAwIBAgIJAKnfUOUcLEqUMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV
-BAMMDGxvY2FsaG9zdC5teTAeFw0xODA1MDgwOTIxMDZaFw0xOTA1MDgwOTIxMDZa
-MBcxFTATBgNVBAMMDGxvY2FsaG9zdC5teTCCASIwDQYJKoZIhvcNAQEBBQADggEP
-ADCCAQoCggEBAMVe3pFwlPCrniAAsDyhoolnwv0gOQ4SX81nA0NggabKbUBJwwfN
-nKP5dvFNHCo100fzoiWbFmZnu9pUMtjeucQzaA38i501rXCkiPmTkE+tDdIJqO8J
-lLV+oaNvFtaAVcRIiuU9fTp/MdZhG3tLj/AXx9dcc1xHRjg/tngepAsvZ2oRoBVU
-ijvkOSCm1xwew+ZTzazLARnLOvHok1tJPepMCVlGaEaL9r1aJ86hMUSg+sli2ayW
-myI4Pt7ZrsyrHpHDYF9ecWWGbmHfgLdaAdyulrPuvtwavl6KtgSuy3SxwigOfdBI
-h4Xw2u6gq4v40OuZGWgkNdJ000ddwurWfosCAwEAAaNQME4wHQYDVR0OBBYEFAd+
-0uv5elelwrjB/0C7EDO7VauqMB8GA1UdIwQYMBaAFAd+0uv5elelwrjB/0C7EDO7
-VauqMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEauDMNWqCIIZXmY
-HLqkoPmy+BDX7N4F2ZuWntes8D/igFhZOYQfD+ksJEv3zgs6N5Qd8HbSCbZR0Hh+
-1g+RjVBu8T67h6+vIDZuu0jORjknUp2XbD+aWG+7UcuUjDY8KF9St50ZniSieiSA
-dV09VrJ/JFwxaeFzgOHnk9oP5eggwZjEZJqSc4qzL0JlhFcxV8R4OVUCjRyHG73p
-cN7nUDL9xN5XZY+6t6+rzdYi4UAhEW0odFVfyXqhOhupSgQkBBdIjxVuov+3h/aV
-D2YweTg6cKtuaISsFmDEPht7cVQuy5z3PPkV6kQBeECA9vTFP3wCxA0n7Iyyn2IK
-8gvWZXk=
------END CERTIFICATE-----
diff --git a/library/cpp/http/simple/ut/https_server/http_server.key b/library/cpp/http/simple/ut/https_server/http_server.key
deleted file mode 100644
index f58ab049fd..0000000000
--- a/library/cpp/http/simple/ut/https_server/http_server.key
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDFXt6RcJTwq54g
-ALA8oaKJZ8L9IDkOEl/NZwNDYIGmym1AScMHzZyj+XbxTRwqNdNH86IlmxZmZ7va
-VDLY3rnEM2gN/IudNa1wpIj5k5BPrQ3SCajvCZS1fqGjbxbWgFXESIrlPX06fzHW
-YRt7S4/wF8fXXHNcR0Y4P7Z4HqQLL2dqEaAVVIo75DkgptccHsPmU82sywEZyzrx
-6JNbST3qTAlZRmhGi/a9WifOoTFEoPrJYtmslpsiOD7e2a7Mqx6Rw2BfXnFlhm5h
-34C3WgHcrpaz7r7cGr5eirYErst0scIoDn3QSIeF8NruoKuL+NDrmRloJDXSdNNH
-XcLq1n6LAgMBAAECggEAdN+wvD8Gc12szRabRcwRC3y+IlYqcwK+aEtPy14iaDoG
-Z8NGEiDXWOIoZMtcmkI1Uq4anlov8YQL4UVqtrFtH5mxTFb39agLhGBqHCAdnJDF
-VlMSDjqGLNNHtBfcVji4kPrEBOtcdH9Na70lIOWl3m62j/jW9xXdpwFTc93xFg14
-Ivtjtv7KHZAPgN0pdgsqen1js6Z3O5tkcy4yFLldBl+8/ZbYSMM+Rh4GbR5qvWfA
-23vBu9EprJKPhFQlNZPbesEKe8EA+SCuLo0RzAZq1E2nZRH0HasKT2hhr/kobkN6
-oLIo2dNgIYL7xMhHLcBt1/08CXKZIqEAfA9Tx/eVgQKBgQD7/oN/XA0pMVCqS8f6
-8Z9VI4NxHJoPFLskrXjXhbxgLgUjuz28cuoyaIKcv8f9Qo7f+hOwR2F3zjwyVexB
-G+0fuyIbqD8Po43F+SBJCVSE3EV5k0AQJJN74a+UuKC39NhGaXsmj+s6ArWrURV5
-thay+308pF5HvYCnmQD3UfOJiQKBgQDIghDarcID6/Q0Nv8xvfd8p9kUu5vX/Tw0
-W22JDDMxpUoYCGXvOEx+IoVzqLOTw+NcEXSmDA41VqXlphYopwZkfNV6kIXVymdu
-oNKisgfe4Hrfrq9BUl5p8gvU/Ev5zY7N4kVirUJgNvRHDElp8h6Ek/KRTv8Q0xRX
-ZW6UqmKGcwKBgDsQZ7/1UnxiO7b+tivicGcjQM7FVnLMeCTbqCRUC1g70SaT35+J
-C82u41ZcOULqU9S5p928jWLoawGdVBfatNSoJxF2ePlwa22IvAGCd1YAzyP02KIw
-AIWb22yvbbRQZlTyqlPajdb2BaDXC4KQpHdlLPCG0jZce4hM+4X8pmmJAoGALW4S
-5YlTGVJf7Wi8n4ecSJk7PVBYujJ9bpt8kP27p7b8t79HYVFPO5EUzaTes09B931Z
-AbpficRNKGBeSu21LBWAxRlzyYHnt5AmyYgu8lfIX2AUA2fnTnfyKFrV2A60GX/4
-GqiJDoXFCUgGZkPemElxP203q5c316l6yaJlWnMCgYAqk1G65THRmdTKcnUEOqo8
-pD3SWuBvbOHYLyg+f0zNAqpnTFbaPVmsWfx3CsX2m8WdH3dD28SGfvepQlWj1yp/
-TmXs14nFUuJWir2VbPgp8W/uZl8bQ0YlI8UPUbN3XbLkVIno+jXuUopcgrXmi7Gb
-Y2QnQfHePgpszWR0o+WiYg==
------END PRIVATE KEY-----
diff --git a/library/cpp/http/simple/ut/https_server/main.go b/library/cpp/http/simple/ut/https_server/main.go
deleted file mode 100644
index 4282810675..0000000000
--- a/library/cpp/http/simple/ut/https_server/main.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package main
-
-import (
- "fmt"
- "log"
- "net/http"
- "os"
-
- "github.com/spf13/cobra"
- "github.com/spf13/pflag"
-)
-
-type Opts struct {
- Port uint16
- KeyFile string
- CertFile string
-}
-
-func handler(writer http.ResponseWriter, request *http.Request) {
- res := "pong.my"
-
- writer.Header().Set("Content-Type", "text/plain")
- writer.WriteHeader(http.StatusOK)
-
- _, _ = writer.Write([]byte(res))
-}
-
-func runServer(opts *Opts) error {
- mainMux := http.NewServeMux()
- mainMux.Handle("/ping", http.HandlerFunc(handler))
-
- server := &http.Server{
- Addr: fmt.Sprintf("localhost:%d", opts.Port),
- Handler: mainMux,
- ErrorLog: log.New(os.Stdout, "", log.LstdFlags),
- }
-
- return server.ListenAndServeTLS(opts.CertFile, opts.KeyFile)
-}
-
-func markFlagRequired(flags *pflag.FlagSet, names ...string) {
- for _, n := range names {
- name := n
- if err := cobra.MarkFlagRequired(flags, name); err != nil {
- panic(err)
- }
- }
-}
-
-func main() {
- opts := Opts{}
-
- cmd := cobra.Command{
- RunE: func(cmd *cobra.Command, args []string) error {
- return runServer(&opts)
- },
- }
-
- flags := cmd.Flags()
- flags.Uint16Var(&opts.Port, "port", 0, "")
- flags.StringVar(&opts.KeyFile, "keyfile", "", "path to key file")
- flags.StringVar(&opts.CertFile, "certfile", "", "path to cert file")
-
- markFlagRequired(flags, "port", "keyfile", "certfile")
-
- if err := cmd.Execute(); err != nil {
- _, _ = fmt.Fprintf(os.Stderr, "Exit with err: %s", err)
- os.Exit(1)
- }
-}
diff --git a/library/cpp/http/simple/ut/https_ut.cpp b/library/cpp/http/simple/ut/https_ut.cpp
deleted file mode 100644
index 3849b9ac9a..0000000000
--- a/library/cpp/http/simple/ut/https_ut.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-#include <library/cpp/http/simple/http_client.h>
-
-#include <library/cpp/http/server/response.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/system/shellcommand.h>
-
-Y_UNIT_TEST_SUITE(Https) {
- using TShellCommandPtr = std::unique_ptr<TShellCommand>;
-
- static TShellCommandPtr start(ui16 port) {
- const TString data = ArcadiaSourceRoot() + "/library/cpp/http/simple/ut/https_server";
-
- const TString command =
- TStringBuilder()
- << BuildRoot() << "/library/cpp/http/simple/ut/https_server/https_server"
- << " --port " << port
- << " --keyfile " << data << "/http_server.key"
- << " --certfile " << data << "/http_server.crt";
-
- auto res = std::make_unique<TShellCommand>(
- command,
- TShellCommandOptions()
- .SetAsync(true)
- .SetLatency(50)
- .SetErrorStream(&Cerr));
-
- res->Run();
-
- i32 tries = 100000;
- while (tries-- > 0) {
- try {
- TKeepAliveHttpClient client("https://localhost", port);
- client.DisableVerificationForHttps();
- client.DoGet("/ping");
- break;
- } catch (const std::exception& e) {
- Cout << "== failed to connect to new server: " << e.what() << Endl;
- Sleep(TDuration::MilliSeconds(1));
- }
- }
-
- return res;
- }
-
- static void get(TKeepAliveHttpClient & client) {
- TStringStream out;
- ui32 code = 0;
-
- UNIT_ASSERT_NO_EXCEPTION(code = client.DoGet("/ping", &out));
- UNIT_ASSERT_VALUES_EQUAL_C(code, 200, out.Str());
- UNIT_ASSERT_VALUES_EQUAL(out.Str(), "pong.my");
- }
-
- Y_UNIT_TEST(keepAlive) {
- TPortManager pm;
- ui16 port = pm.GetPort(443);
- TShellCommandPtr httpsServer = start(port);
-
- TKeepAliveHttpClient client("https://localhost",
- port,
- TDuration::Seconds(40),
- TDuration::Seconds(40));
- client.DisableVerificationForHttps();
-
- get(client);
- get(client);
-
- httpsServer->Terminate().Wait();
- httpsServer = start(port);
-
- get(client);
- }
-
- static void get(TSimpleHttpClient & client) {
- TStringStream out;
-
- UNIT_ASSERT_NO_EXCEPTION_C(client.DoGet("/ping", &out), out.Str());
- UNIT_ASSERT_VALUES_EQUAL(out.Str(), "pong.my");
- }
-
- Y_UNIT_TEST(simple) {
- TPortManager pm;
- ui16 port = pm.GetPort(443);
- TShellCommandPtr httpsServer = start(port);
-
- TSimpleHttpClient client("https://localhost",
- port,
- TDuration::Seconds(40),
- TDuration::Seconds(40));
-
- get(client);
- get(client);
- }
-}
diff --git a/library/cpp/openssl/big_integer/CMakeLists.txt b/library/cpp/openssl/big_integer/CMakeLists.txt
deleted file mode 100644
index 55d94de253..0000000000
--- a/library/cpp/openssl/big_integer/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-find_package(OpenSSL REQUIRED)
-
-add_library(cpp-openssl-big_integer)
-target_link_libraries(cpp-openssl-big_integer PUBLIC
- contrib-libs-cxxsupp
- yutil
- OpenSSL::OpenSSL
-)
-target_sources(cpp-openssl-big_integer PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/openssl/big_integer/big_integer.cpp
-)
diff --git a/library/cpp/openssl/big_integer/big_integer.cpp b/library/cpp/openssl/big_integer/big_integer.cpp
deleted file mode 100644
index de59f84499..0000000000
--- a/library/cpp/openssl/big_integer/big_integer.cpp
+++ /dev/null
@@ -1,61 +0,0 @@
-#include "big_integer.h"
-
-#include <util/generic/yexception.h>
-#include <util/generic/scope.h>
-#include <util/stream/output.h>
-
-#include <contrib/libs/openssl/include/openssl/bn.h>
-
-using namespace NOpenSsl;
-
-TBigInteger::~TBigInteger() noexcept {
- BN_free(Impl_);
-}
-
-TBigInteger TBigInteger::FromULong(ui64 value) {
- TBigInteger result(BN_new());
-
- Y_ENSURE(result.Impl(), "BN_new() failed");
- Y_ENSURE(BN_set_word(result.Impl(), value) == 1, "BN_set_word() failed");
-
- return result;
-}
-
-TBigInteger TBigInteger::FromRegion(const void* ptr, size_t len) {
- auto result = BN_bin2bn((ui8*)(ptr), len, nullptr);
-
- Y_ENSURE(result, "BN_bin2bn() failed");
-
- return result;
-}
-
-int TBigInteger::Compare(const TBigInteger& a, const TBigInteger& b) noexcept {
- return BN_cmp(a.Impl(), b.Impl());
-}
-
-size_t TBigInteger::NumBytes() const noexcept {
- return BN_num_bytes(Impl_);
-}
-
-size_t TBigInteger::ToRegion(void* to) const noexcept {
- const auto ret = BN_bn2bin(Impl_, (unsigned char*)to);
-
- Y_VERIFY(ret >= 0, "it happens");
-
- return ret;
-}
-
-TString TBigInteger::ToDecimalString() const {
- auto res = BN_bn2dec(Impl_);
-
- Y_DEFER {
- OPENSSL_free(res);
- };
-
- return res;
-}
-
-template <>
-void Out<TBigInteger>(IOutputStream& out, const TBigInteger& bi) {
- out << bi.ToDecimalString();
-}
diff --git a/library/cpp/openssl/big_integer/big_integer.h b/library/cpp/openssl/big_integer/big_integer.h
deleted file mode 100644
index 07763c5e13..0000000000
--- a/library/cpp/openssl/big_integer/big_integer.h
+++ /dev/null
@@ -1,57 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/utility.h>
-#include <util/generic/string.h>
-
-struct bignum_st;
-
-namespace NOpenSsl {
- class TBigInteger {
- inline TBigInteger(bignum_st* impl) noexcept
- : Impl_(impl)
- {
- }
-
- static int Compare(const TBigInteger& a, const TBigInteger& b) noexcept;
-
- public:
- inline TBigInteger(TBigInteger&& other) noexcept {
- Swap(other);
- }
-
- ~TBigInteger() noexcept;
-
- static TBigInteger FromULong(ui64 value);
- static TBigInteger FromRegion(const void* ptr, size_t len);
-
- inline const bignum_st* Impl() const noexcept {
- return Impl_;
- }
-
- inline bignum_st* Impl() noexcept {
- return Impl_;
- }
-
- inline void Swap(TBigInteger& other) noexcept {
- DoSwap(Impl_, other.Impl_);
- }
-
- inline friend bool operator==(const TBigInteger& a, const TBigInteger& b) noexcept {
- return Compare(a, b) == 0;
- }
-
- inline friend bool operator!=(const TBigInteger& a, const TBigInteger& b) noexcept {
- return !(a == b);
- }
-
- size_t NumBytes() const noexcept;
- size_t ToRegion(void* to) const noexcept;
-
- TString ToDecimalString() const;
-
- private:
- bignum_st* Impl_ = nullptr;
- };
-}
diff --git a/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp b/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp
deleted file mode 100644
index 8a0050f531..0000000000
--- a/library/cpp/openssl/big_integer/ut/big_integer_ut.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-#include "big_integer.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/byteorder.h>
-#include <util/stream/str.h>
-
-Y_UNIT_TEST_SUITE(BigInteger) {
- using NOpenSsl::TBigInteger;
-
- Y_UNIT_TEST(Initialization) {
- constexpr ui64 testVal = 12345678900;
- const auto fromULong = TBigInteger::FromULong(testVal);
-
- const ui64 testArea = HostToInet(testVal); // transform to big-endian
- const auto fromRegion = TBigInteger::FromRegion(&testArea, sizeof(testArea));
- UNIT_ASSERT(fromULong == fromRegion);
- UNIT_ASSERT_VALUES_EQUAL(fromULong, fromRegion);
-
- const auto fromULongOther = TBigInteger::FromULong(22345678900);
- UNIT_ASSERT(fromULong != fromULongOther);
- }
-
- Y_UNIT_TEST(Decimal) {
- UNIT_ASSERT_VALUES_EQUAL(TBigInteger::FromULong(123456789).ToDecimalString(), "123456789");
- }
-
- Y_UNIT_TEST(Region) {
- const auto v1 = TBigInteger::FromULong(1234567890);
- char buf[1024];
- const auto v2 = TBigInteger::FromRegion(buf, v1.ToRegion(buf));
-
- UNIT_ASSERT_VALUES_EQUAL(v1, v2);
- }
-
- Y_UNIT_TEST(Output) {
- TStringStream ss;
-
- ss << TBigInteger::FromULong(123456789);
-
- UNIT_ASSERT_VALUES_EQUAL(ss.Str(), "123456789");
- }
-}
diff --git a/library/cpp/openssl/crypto/CMakeLists.txt b/library/cpp/openssl/crypto/CMakeLists.txt
deleted file mode 100644
index 5f6664acd8..0000000000
--- a/library/cpp/openssl/crypto/CMakeLists.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-find_package(OpenSSL REQUIRED)
-
-add_library(cpp-openssl-crypto)
-target_link_libraries(cpp-openssl-crypto PUBLIC
- contrib-libs-cxxsupp
- yutil
- OpenSSL::OpenSSL
- cpp-openssl-big_integer
- cpp-openssl-init
-)
-target_sources(cpp-openssl-crypto PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/sha.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/openssl/crypto/rsa.cpp
-)
diff --git a/library/cpp/openssl/crypto/rsa.cpp b/library/cpp/openssl/crypto/rsa.cpp
deleted file mode 100644
index 350e0c0815..0000000000
--- a/library/cpp/openssl/crypto/rsa.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-#include "rsa.h"
-
-#include <library/cpp/openssl/big_integer/big_integer.h>
-#include <library/cpp/openssl/init/init.h>
-
-#include <util/generic/yexception.h>
-#include <util/generic/buffer.h>
-
-#include <contrib/libs/openssl/include/openssl/bn.h>
-#include <contrib/libs/openssl/include/openssl/rsa.h>
-
-using namespace NOpenSsl;
-using namespace NOpenSsl::NRsa;
-
-namespace {
- struct TInit {
- inline TInit() {
- InitOpenSSL();
- }
- } INIT;
-}
-
-TPublicKey::TPublicKey(const TBigInteger& e, const TBigInteger& n)
- : Key_(RSA_new())
-{
- Y_ENSURE(Key_, "RSA_new() failed");
-
- RSA_set0_key(Key_, BN_dup(n.Impl()), BN_dup(e.Impl()), nullptr);
-}
-
-TPublicKey::~TPublicKey() noexcept {
- RSA_free(Key_);
-}
-
-size_t TPublicKey::OutputLength() const noexcept {
- return RSA_size(Key_);
-}
-
-size_t TPublicKey::EncryptNoPad(void* dst, const void* src, size_t size) const {
- auto len = RSA_public_encrypt(size, (const ui8*)src, (ui8*)dst, Key_, RSA_NO_PADDING);
-
- Y_ENSURE(len >= 0, "RSA_public_encrypt() failed");
-
- return len;
-}
-
-TBigInteger TPublicKey::EncryptNoPad(const TBigInteger& src) const {
- const auto len1 = OutputLength();
- const auto len2 = src.NumBytes();
- TBuffer buf(len1 + len2);
-
- char* buf1 = (char*)buf.Data();
- char* buf2 = buf1 + len1;
-
- return TBigInteger::FromRegion(buf1, EncryptNoPad(buf1, buf2, src.ToRegion(buf2)));
-}
diff --git a/library/cpp/openssl/crypto/rsa.h b/library/cpp/openssl/crypto/rsa.h
deleted file mode 100644
index 3bf9e4a233..0000000000
--- a/library/cpp/openssl/crypto/rsa.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#pragma once
-
-#include <util/generic/utility.h>
-#include <util/generic/noncopyable.h>
-
-struct rsa_st;
-
-namespace NOpenSsl {
- class TBigInteger;
-
- namespace NRsa {
- class TPublicKey: public TNonCopyable {
- public:
- inline TPublicKey(TPublicKey&& other) noexcept {
- Swap(other);
- }
-
- TPublicKey(const TBigInteger& e, const TBigInteger& n);
- ~TPublicKey() noexcept;
-
- size_t OutputLength() const noexcept;
-
- TBigInteger EncryptNoPad(const TBigInteger& src) const;
- size_t EncryptNoPad(void* dst, const void* src, size_t size) const;
-
- inline void Swap(TPublicKey& other) noexcept {
- DoSwap(Key_, other.Key_);
- }
-
- private:
- rsa_st* Key_ = nullptr;
- };
- };
-}
diff --git a/library/cpp/openssl/crypto/sha.cpp b/library/cpp/openssl/crypto/sha.cpp
deleted file mode 100644
index 8e7e88ccde..0000000000
--- a/library/cpp/openssl/crypto/sha.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
-#include "sha.h"
-
-#include <util/generic/yexception.h>
-
-#include <contrib/libs/openssl/include/openssl/sha.h>
-
-namespace NOpenSsl {
- namespace NSha1 {
- static_assert(DIGEST_LENGTH == SHA_DIGEST_LENGTH);
-
- TDigest Calc(const void* data, size_t dataSize) {
- TDigest digest;
- Y_ENSURE(SHA1(static_cast<const ui8*>(data), dataSize, digest.data()) != nullptr);
- return digest;
- }
-
- TCalcer::TCalcer()
- : Context{new SHAstate_st} {
- Y_ENSURE(SHA1_Init(Context.Get()) == 1);
- }
-
- TCalcer::~TCalcer() {
- }
-
- void TCalcer::Update(const void* data, size_t dataSize) {
- Y_ENSURE(SHA1_Update(Context.Get(), data, dataSize) == 1);
- }
-
- TDigest TCalcer::Final() {
- TDigest digest;
- Y_ENSURE(SHA1_Final(digest.data(), Context.Get()) == 1);
- return digest;
- }
- }
- namespace NSha256 {
- static_assert(DIGEST_LENGTH == SHA256_DIGEST_LENGTH);
-
- TDigest Calc(const void* data, size_t dataSize) {
- TDigest digest;
- Y_ENSURE(SHA256(static_cast<const ui8*>(data), dataSize, digest.data()) != nullptr);
- return digest;
- }
-
- TCalcer::TCalcer()
- : Context{new SHA256state_st} {
- Y_ENSURE(SHA256_Init(Context.Get()) == 1);
- }
-
- TCalcer::~TCalcer() {
- }
-
- void TCalcer::Update(const void* data, size_t dataSize) {
- Y_ENSURE(SHA256_Update(Context.Get(), data, dataSize) == 1);
- }
-
- TDigest TCalcer::Final() {
- TDigest digest;
- Y_ENSURE(SHA256_Final(digest.data(), Context.Get()) == 1);
- return digest;
- }
- }
-}
diff --git a/library/cpp/openssl/crypto/sha.h b/library/cpp/openssl/crypto/sha.h
deleted file mode 100644
index dbc2dfa526..0000000000
--- a/library/cpp/openssl/crypto/sha.h
+++ /dev/null
@@ -1,78 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/generic/strbuf.h>
-#include <util/system/types.h>
-
-#include <array>
-
-struct SHAstate_st;
-struct SHA256state_st;
-
-namespace NOpenSsl::NSha1 {
- constexpr size_t DIGEST_LENGTH = 20;
- using TDigest = std::array<ui8, DIGEST_LENGTH>;
-
- // not fragmented input
- TDigest Calc(const void* data, size_t dataSize);
-
- inline TDigest Calc(TStringBuf s) {
- return Calc(s.data(), s.length());
- }
-
- // fragmented input
- class TCalcer {
- public:
- TCalcer();
- ~TCalcer();
- void Update(const void* data, size_t dataSize);
-
- void Update(TStringBuf s) {
- Update(s.data(), s.length());
- }
-
- template <typename T>
- void UpdateWithPodValue(const T& value) {
- Update(&value, sizeof(value));
- }
-
- TDigest Final();
-
- private:
- THolder<SHAstate_st> Context;
- };
-}
-
-namespace NOpenSsl::NSha256 {
- constexpr size_t DIGEST_LENGTH = 32;
- using TDigest = std::array<ui8, DIGEST_LENGTH>;
-
- // not fragmented input
- TDigest Calc(const void* data, size_t dataSize);
-
- inline TDigest Calc(TStringBuf s) {
- return Calc(s.data(), s.length());
- }
-
- // fragmented input
- class TCalcer {
- public:
- TCalcer();
- ~TCalcer();
- void Update(const void* data, size_t dataSize);
-
- void Update(TStringBuf s) {
- Update(s.data(), s.length());
- }
-
- template <typename T>
- void UpdateWithPodValue(const T& value) {
- Update(&value, sizeof(value));
- }
-
- TDigest Final();
-
- private:
- THolder<SHA256state_st> Context;
- };
-}
diff --git a/library/cpp/openssl/crypto/sha_ut.cpp b/library/cpp/openssl/crypto/sha_ut.cpp
deleted file mode 100644
index 4b6117f960..0000000000
--- a/library/cpp/openssl/crypto/sha_ut.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
-#include <library/cpp/testing/unittest/registar.h>
-
-#include "sha.h"
-
-constexpr TStringBuf SomeAlignedShaTestData = "some _aligned_ test data for SHA-family: align align align align";
-
-Y_UNIT_TEST_SUITE(SHA){
- Y_UNIT_TEST(CheckOfTestDataAlignment){
- UNIT_ASSERT_VALUES_EQUAL(SomeAlignedShaTestData.size() % sizeof(ui32), 0);
- }
-
- Y_UNIT_TEST(Sha1Value) {
- // bash$ echo -n $SomeAlignedShaTestData | sha1sum
- const TStringBuf precalculatedDigest =
- "\xA2\x29\x8E\xE2\xEA\x06\x27\x45"
- "\x27\xC7\x78\x87\x16\x21\x8A\xA5"
- "\x0D\xBA\xBA\xB2"sv;
-
- auto digest = NOpenSsl::NSha1::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size());
-
- UNIT_ASSERT_VALUES_EQUAL(precalculatedDigest.size(), digest.size());
- UNIT_ASSERT_VALUES_EQUAL(memcmp(precalculatedDigest.data(), digest.data(), digest.size()), 0);
- }
-
- Y_UNIT_TEST(Sha256Value) {
- // bash$ echo -n $SomeAlignedShaTestData | sha256sum
- const TStringBuf precalculatedDigest =
- "\xED\x64\x0D\x43\xF7\x6D\x71\x98"
- "\x39\x19\xF6\xE6\x70\x21\x82\x11"
- "\xEF\x3B\xF0\xF4\x35\xBF\x42\xAB"
- "\x1C\x5C\x01\xCD\x20\x33\xD2\xFA"sv;
-
- auto digest = NOpenSsl::NSha256::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size());
-
- UNIT_ASSERT_VALUES_EQUAL(precalculatedDigest.size(), digest.size());
- UNIT_ASSERT_VALUES_EQUAL(memcmp(precalculatedDigest.data(), digest.data(), digest.size()), 0);
- }
-
- Y_UNIT_TEST(FragmentedEqualNotFragmented) {
- const char* head = SomeAlignedShaTestData.data();
- const char* current = head;
- NOpenSsl::NSha1::TCalcer sha;
- int intValue;
- std::copy_n(current, sizeof(intValue), (char*)&intValue);
- current += sizeof(intValue);
- sha.UpdateWithPodValue(intValue);
- double doubleValue;
- std::copy_n(current, sizeof(doubleValue), (char*)&doubleValue);
- current += sizeof(doubleValue);
- sha.UpdateWithPodValue(doubleValue);
- char str[7];
- std::copy_n(current, std::size(str), str);
- current += std::size(str);
- sha.UpdateWithPodValue(str);
- sha.Update(current, SomeAlignedShaTestData.size() - (current - head));
- auto fragmentedDigest = sha.Final();
-
- auto notFragmentedDigest = NOpenSsl::NSha1::Calc(SomeAlignedShaTestData.data(), SomeAlignedShaTestData.size());
-
- UNIT_ASSERT_VALUES_EQUAL(memcmp(fragmentedDigest.data(), notFragmentedDigest.data(), notFragmentedDigest.size()), 0);
- }
-}; // UNITTEST_SIMPLE_SUITE(SHA)
diff --git a/library/cpp/openssl/crypto/ut/rsa_ut.cpp b/library/cpp/openssl/crypto/ut/rsa_ut.cpp
deleted file mode 100644
index 99f0377432..0000000000
--- a/library/cpp/openssl/crypto/ut/rsa_ut.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "rsa.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/openssl/big_integer/big_integer.h>
-
-#include <util/system/byteorder.h>
-
-using namespace NOpenSsl;
-using namespace NOpenSsl::NRsa;
-
-Y_UNIT_TEST_SUITE(Rsa) {
- Y_UNIT_TEST(Encrypt) {
- // example from Ru.Wikipedia
- const auto originData = TBigInteger::FromULong(111111);
-
- const auto n = TBigInteger::FromULong(3);
- const auto e = TBigInteger::FromULong(9173503);
-
- // check key reuse
- for (size_t i = 0; i < 10; ++i) {
- UNIT_ASSERT_VALUES_EQUAL(TBigInteger::FromULong(4051753), TPublicKey(n, e).EncryptNoPad(originData));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(originData, TBigInteger::FromULong(111111));
- UNIT_ASSERT_VALUES_EQUAL(n, TBigInteger::FromULong(3));
- UNIT_ASSERT_VALUES_EQUAL(e, TBigInteger::FromULong(9173503));
- }
-};
diff --git a/library/cpp/streams/lzop/CMakeLists.txt b/library/cpp/streams/lzop/CMakeLists.txt
deleted file mode 100644
index 758a7f7d91..0000000000
--- a/library/cpp/streams/lzop/CMakeLists.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-streams-lzop)
-target_link_libraries(cpp-streams-lzop PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-minilzo
-)
-target_sources(cpp-streams-lzop PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/streams/lzop/lzop.cpp
-)
diff --git a/library/cpp/streams/lzop/lzop.cpp b/library/cpp/streams/lzop/lzop.cpp
deleted file mode 100644
index 5d1ad33f70..0000000000
--- a/library/cpp/streams/lzop/lzop.cpp
+++ /dev/null
@@ -1,827 +0,0 @@
-#include "lzop.h"
-
-#include <util/generic/buffer.h>
-
-#include <util/system/info.h>
-
-#include <contrib/libs/minilzo/minilzo.h>
-
-// See https://svn.yandex.ru/statbox/packages/yandex/statbox-binaries/include/Statbox/LZOP.h
-// https://github.yandex-team.ru/logbroker/push-client/blob/c820971a769df920d6ea9152a053474e75986914/src/lzo.c
-// https://github.yandex-team.ru/logbroker/push-client/blob/c820971a769df920d6ea9152a053474e75986914/src/lzo.h
-// As the source for the inspiration.
-////////////////////////////////////////////////////////////////////////////////
-
-namespace NPrivate {
- namespace NLzop {
- static unsigned const char MAGIC[9] =
- {
- 0x89, 0x4c, 0x5a, 0x4f,
- 0x00,
- 0x0d, 0x0a, 0x1a, 0x0a};
-
- // 32-bit Version.
- inline unsigned int RoundUpToPow2(unsigned int x) {
- x -= 1;
- x |= (x >> 1);
- x |= (x >> 2);
- x |= (x >> 4);
- x |= (x >> 8);
- x |= (x >> 16);
- return x + 1;
- }
-
- inline unsigned char* Get8(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) {
- *v = 0;
- *v |= (unsigned)(*p++);
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 1), 1);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 1), 1);
-
- return p;
- }
-
- inline unsigned char* Put8(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) {
- *p++ = v & 0xff;
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 1), 1);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 1), 1);
-
- return p;
- }
-
- inline unsigned char* Get16(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) {
- *v = 0;
- *v |= (unsigned)(*p++) << 8;
- *v |= (unsigned)(*p++);
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 2), 2);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 2), 2);
-
- return p;
- }
-
- inline unsigned char* Put16(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) {
- *p++ = (v >> 8) & 0xff;
- *p++ = (v)&0xff;
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 2), 2);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 2), 2);
-
- return p;
- }
-
- inline unsigned char* Get32(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) {
- *v = 0;
- *v |= (unsigned)(*p++) << 24;
- *v |= (unsigned)(*p++) << 16;
- *v |= (unsigned)(*p++) << 8;
- *v |= (unsigned)(*p++);
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 4), 4);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 4), 4);
-
- return p;
- }
-
- inline unsigned char* Put32(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) {
- *p++ = (v >> 24) & 0xff;
- *p++ = (v >> 16) & 0xff;
- *p++ = (v >> 8) & 0xff;
- *p++ = (v)&0xff;
-
- *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 4), 4);
- *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 4), 4);
-
- return p;
- }
-
- enum ELzoFlag {
- LZO_ADLER32_D = 0x00000001L,
- LZO_ADLER32_C = 0x00000002L,
- LZO_STDIN = 0x00000004L,
- LZO_STDOUT = 0x00000008L,
- LZO_NAME_DEFAULT = 0x00000010L,
- LZO_DOSISH = 0x00000020L,
- LZO_H_EXTRA_FIELD = 0x00000040L,
- LZO_H_GMTDIFF = 0x00000080L,
- LZO_CRC32_D = 0x00000100L,
- LZO_CRC32_C = 0x00000200L,
- LZO_MULTIPART = 0x00000400L,
- LZO_H_FILTER = 0x00000800L,
- LZO_H_CRC32 = 0x00001000L,
- LZO_H_PATH = 0x00002000L,
- LZO_MASK = 0x00003FFFL
- };
-
- enum ELzoResult {
- LZO_END_OF_STREAM = 0,
- LZO_MORE_DATA = 1,
- LZO_OK = 2,
- LZO_WRONG_MAGIC = -12,
- LZO_VERSION_TOO_LOW = -13,
- LZO_EXTRACT_VERSION_TOO_HIGH = -14,
- LZO_EXTRACT_VERSION_TOO_LOW = -15,
- LZO_WRONG_CHECKSUM = -16,
- LZO_WRONG_METHOD = -18,
- LZO_COMPRESS_ERROR = -1,
- LZO_WRONG_DST_LEN = -2,
- LZO_DST_LEN_TOO_BIG = -3,
- LZO_WRONG_SRC_LEN = -4,
- LZO_INVALID_SRC_ADLER32 = -5,
- LZO_INVALID_SRC_CRC32 = -6,
- LZO_DECOMPRESS_ERROR = -7,
- LZO_INVALID_DST_ADLER32 = -8,
- LZO_INVALID_DST_CRC32 = -9,
- };
-
- // XXX(sandello): I don't really know where this comes from.
- struct THeader {
- unsigned Version;
- unsigned LibVersion;
- unsigned VersionNeededToExtract;
- unsigned char Method;
- unsigned char Level;
-
- ui32 Flags;
- ui32 Filter;
- ui32 Mode;
- ui32 MTimeLow;
- ui32 MTimeHigh;
-
- ui32 HeaderChecksum;
-
- ui32 ExtraFieldLen;
- ui32 ExtraFieldChecksum;
-
- const unsigned char* MethodName;
-
- char Name[255 + 1];
- };
-
- }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TLzopCompress::TImpl
- : public IOutputStream {
-public:
- inline TImpl(IOutputStream* slave, ui16 blockSize)
- : Slave(slave)
- , HeaderWasWritten(false)
- , UncompressedBuffer(blockSize)
- , CompressedBuffer(8 + 4 * blockSize)
- {
- ResetHeader();
- }
-
-protected:
- void DoWrite(const void* buffer, size_t length) override;
- void DoFlush() override;
- void DoFinish() override;
-
-private:
- IOutputStream* Slave;
-
- NPrivate::NLzop::THeader Header;
- bool HeaderWasWritten;
-
- TBuffer UncompressedBuffer;
- TBuffer CompressedBuffer;
-
- void EnsureCompressedSpace(size_t demand);
- void EnsureUncompressedSpace(size_t demand);
-
- void ProduceHeader();
- void ProduceData();
- void ProduceTrailer();
- void ResetHeader();
-};
-
-void TLzopCompress::TImpl::DoWrite(const void* buffer, size_t length) {
- const char* data = (const char*)buffer;
- while (length > 0) {
- size_t bytesToFit = Min(UncompressedBuffer.Capacity(), length);
- size_t bytesToWrite = Min(UncompressedBuffer.Avail(), length);
- if (bytesToWrite > 0) {
- UncompressedBuffer.Append(data, bytesToWrite);
- data += bytesToWrite;
- length -= bytesToWrite;
- } else {
- EnsureUncompressedSpace(bytesToFit);
- }
- }
-}
-
-void TLzopCompress::TImpl::DoFlush() {
- EnsureUncompressedSpace(UncompressedBuffer.Capacity());
- EnsureCompressedSpace(CompressedBuffer.Capacity());
-}
-
-void TLzopCompress::TImpl::DoFinish() {
- EnsureUncompressedSpace(UncompressedBuffer.Capacity());
- ProduceTrailer();
- Flush();
-}
-
-void TLzopCompress::TImpl::EnsureCompressedSpace(size_t demand) {
- Y_ASSERT(demand <= CompressedBuffer.Capacity());
- if (CompressedBuffer.Avail() < demand) {
- Slave->Write(CompressedBuffer.Data(), CompressedBuffer.Size());
- CompressedBuffer.Clear();
- }
- Y_ASSERT(demand <= CompressedBuffer.Avail());
-}
-
-void TLzopCompress::TImpl::EnsureUncompressedSpace(size_t demand) {
- Y_ASSERT(demand <= UncompressedBuffer.Capacity());
- if (UncompressedBuffer.Avail() < demand) {
- ProduceData();
- }
- Y_ASSERT(demand <= UncompressedBuffer.Avail());
-}
-
-void TLzopCompress::TImpl::ResetHeader() {
- ::memset(&Header, 0, sizeof(Header));
-}
-
-void TLzopCompress::TImpl::ProduceHeader() {
- using namespace NPrivate::NLzop;
-
- ui32 adler32 = 1;
- ui32 crc32 = 0;
-
- unsigned char* p;
- unsigned char* pb;
-
- EnsureCompressedSpace(sizeof(MAGIC) + sizeof(Header));
- pb = p = (unsigned char*)CompressedBuffer.Pos();
-
- // Magic.
- ::memcpy(p, MAGIC, sizeof(MAGIC));
- p += sizeof(MAGIC);
-
- // .Version
- p = Put16(p, 0x1030U, &adler32, &crc32);
- // .LibVersion
- p = Put16(p, lzo_version() & 0xFFFFU, &adler32, &crc32);
- // .VersionNeededToExtract
- p = Put16(p, 0x0900, &adler32, &crc32);
- // .Method
- // XXX(sandello): Method deviates from Statbox' implementation.
- // In compatibility we trust.
- p = Put8(p, 2, &adler32, &crc32); // 1 = LZO1X_1, 2 = LZO1X_1_15
- // .Level
- p = Put8(p, 3, &adler32, &crc32);
- // .Flags
- p = Put32(p, 0, &adler32, &crc32);
- // .Mode
- p = Put32(p, 0644, &adler32, &crc32);
- // .MTimeLow
- p = Put32(p, 0, &adler32, &crc32);
- // .MTimeHigh
- p = Put32(p, 0, &adler32, &crc32);
- // .Name
- p = Put8(p, 0, &adler32, &crc32);
- // .HeaderChecksum
- p = Put32(p, adler32, &adler32, &crc32);
-
- CompressedBuffer.Proceed(CompressedBuffer.Size() + (p - pb));
-}
-
-void TLzopCompress::TImpl::ProduceTrailer() {
- using namespace NPrivate::NLzop;
-
- ui32 adler32 = 1;
- ui32 crc32 = 0;
-
- unsigned char* p;
- unsigned char* pb;
-
- EnsureCompressedSpace(4);
- pb = p = (unsigned char*)CompressedBuffer.Pos();
-
- p = Put32(p, 0, &adler32, &crc32);
-
- CompressedBuffer.Proceed(CompressedBuffer.Size() + (p - pb));
-}
-
-void TLzopCompress::TImpl::ProduceData() {
- using namespace NPrivate::NLzop;
-
- ui32 srcLen = (ui32)UncompressedBuffer.Size();
- ui32 dstLen;
-
- ui32 adler32 = 1;
- ui32 crc32 = 0;
-
- unsigned char* p;
- unsigned char* pb;
-
- lzo_uint result;
-
- // See include/lzo/lzo1x.h from lzo-2.06.
- // const size_t LZO1X_1_MEM_COMPRESS = (lzo_uint32)(16384L * lzo_sizeof_dict_t);
- unsigned char scratch[LZO1X_1_MEM_COMPRESS];
-
- if (!HeaderWasWritten) {
- ProduceHeader();
- HeaderWasWritten = true;
- }
-
- EnsureCompressedSpace(8 + 4 * srcLen);
- pb = p = (unsigned char*)CompressedBuffer.Pos();
-
- p = Put32(p, srcLen, &adler32, &crc32);
- p += 4;
-
- // XXX(sandello): Used compression Method deviates from Statbox's implementation.
- // Here we use |lzo1x_1_compress| (implemented in minilzo) whilst Statbox
- // uses |lzo1x_1_15_compress|.
- if (lzo1x_1_compress(
- (unsigned char*)UncompressedBuffer.Data(),
- UncompressedBuffer.Size(),
- p,
- &result,
- scratch) != LZO_E_OK)
- {
- ythrow yexception() << "LZOP Error: " << (int)LZO_COMPRESS_ERROR;
- }
-
- dstLen = result;
-
- if (dstLen < srcLen) {
- Put32(pb + 4, dstLen, &adler32, &crc32);
- /**/
- result = dstLen;
- } else {
- Put32(pb + 4, srcLen, &adler32, &crc32);
- ::memcpy(p, UncompressedBuffer.Data(), UncompressedBuffer.Size());
- result = srcLen;
- }
-
- result += 4 + 4; // srcLen + dstLen + (adler32|crc32, disabled)
-
- UncompressedBuffer.Clear();
- CompressedBuffer.Proceed(CompressedBuffer.Size() + result);
-}
-
-TLzopCompress::TLzopCompress(IOutputStream* slave, ui16 maxBlockSize)
- : Impl_(new TImpl(slave, maxBlockSize))
-{
-}
-
-TLzopCompress::~TLzopCompress() {
- try {
- Finish();
- } catch (...) {
- }
-}
-
-void TLzopCompress::DoWrite(const void* buffer, size_t length) {
- if (!Impl_) {
- ythrow yexception() << "Stream is dead";
- }
- Impl_->Write((const char*)buffer, length);
-}
-
-void TLzopCompress::DoFlush() {
- if (!Impl_) {
- ythrow yexception() << "Stream is dead";
- }
- Impl_->Flush();
-}
-
-void TLzopCompress::DoFinish() {
- THolder<TImpl> impl(Impl_.Release());
- if (!!impl) {
- impl->Finish();
- }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-class TLzopDecompress::TImpl
- : public IInputStream {
-public:
- inline TImpl(IInputStream* slave, ui32 initialBufferSize)
- : Slave(slave)
- , Exhausted(false)
- , Hint(0)
- , InputData(NPrivate::NLzop::RoundUpToPow2(initialBufferSize))
- , OutputData(NPrivate::NLzop::RoundUpToPow2(initialBufferSize))
- , InputOffset(0)
- , OutputOffset(0)
- {
- ResetHeader();
- }
-
-protected:
- size_t DoRead(void* buffer, size_t length) override;
-
-private:
- IInputStream* Slave;
- bool Exhausted;
- unsigned int Hint;
-
- NPrivate::NLzop::THeader Header;
-
- TBuffer InputData;
- TBuffer OutputData;
-
- size_t InputOffset;
- size_t OutputOffset;
-
- void Trim(TBuffer& buffer, size_t& length);
-
- int ConsumeHeader();
- int ConsumeData();
- void ResetHeader();
-};
-
-size_t TLzopDecompress::TImpl::DoRead(void* buffer, size_t length) {
- size_t bytesRead = 0;
- size_t bytesAvailable = 0;
-
- do {
- bytesAvailable = Min(OutputData.Size() - OutputOffset, length);
- if (!bytesAvailable && !Exhausted) {
- int rv;
- while ((rv = ConsumeData()) == NPrivate::NLzop::LZO_MORE_DATA) {
- if (Hint) {
- InputData.Reserve(NPrivate::NLzop::RoundUpToPow2(Hint));
- Hint = 0;
- } else if (InputData.Avail() == 0) {
- InputData.Reserve(2 * InputData.Capacity());
- }
-
- size_t tmp = Slave->Load(InputData.Pos(), InputData.Avail());
- if (tmp) {
- InputData.Advance(tmp);
- } else {
- Exhausted = true;
- break;
- }
- }
-
- Trim(InputData, InputOffset);
-
- switch (rv) {
- case NPrivate::NLzop::LZO_OK:
- case NPrivate::NLzop::LZO_MORE_DATA:
- break;
- case NPrivate::NLzop::LZO_END_OF_STREAM:
- ResetHeader();
- break;
- default:
- ythrow yexception() << "LZOP Error: " << rv;
- break;
- }
- } else if (bytesAvailable) {
- ::memcpy(
- (char*)buffer + bytesRead,
- OutputData.Data() + OutputOffset,
- bytesAvailable);
- bytesRead += bytesAvailable;
- OutputOffset += bytesAvailable;
-
- Trim(OutputData, OutputOffset);
- } else {
- break;
- }
- } while (!bytesRead);
-
- return bytesRead;
-}
-
-void TLzopDecompress::TImpl::ResetHeader() {
- ::memset(&Header, 0, sizeof(Header));
-}
-
-void TLzopDecompress::TImpl::Trim(TBuffer& buffer, size_t& length) {
- size_t remaining = buffer.Size() - length;
- ::memmove(
- buffer.Data(),
- buffer.Data() + length,
- remaining);
- buffer.Resize(remaining);
- length = 0;
-}
-
-int TLzopDecompress::TImpl::ConsumeHeader() {
- using namespace NPrivate::NLzop;
-
- THeader* h = &Header;
-
- ui32 adler32 = 1;
- ui32 crc32 = 0;
- ui32 checksum;
-
- unsigned tmp;
-
- unsigned char* p;
- unsigned char* pb;
- unsigned char* pe;
-
- pb = p = (unsigned char*)InputData.Data() + InputOffset;
- pe = (unsigned char*)InputData.Pos();
-
- // Magic.
- if (pe < p + sizeof(MAGIC))
- return LZO_MORE_DATA;
- if (memcmp(MAGIC, p, sizeof(MAGIC)) != 0) {
- return LZO_WRONG_MAGIC;
- }
- p += sizeof(MAGIC);
-
- // .Version
- if (pe - p < 2)
- return LZO_MORE_DATA;
- p = Get16(p, &h->Version, &adler32, &crc32);
- if (h->Version < 0x0900) {
- return LZO_VERSION_TOO_LOW;
- }
-
- // .LibVersion, .VersionNeededToExtract
- if (pe - p < 2)
- return LZO_MORE_DATA;
- p = Get16(p, &h->LibVersion, &adler32, &crc32);
- if (h->Version >= 0x0940) {
- if (pe - p < 2)
- return LZO_MORE_DATA;
- p = Get16(p, &h->VersionNeededToExtract, &adler32, &crc32);
- if (h->VersionNeededToExtract > 0x1030) {
- return LZO_EXTRACT_VERSION_TOO_HIGH;
- }
- if (h->VersionNeededToExtract < 0x0900) {
- return LZO_EXTRACT_VERSION_TOO_LOW;
- }
- }
-
- // .Method, .Level
- if (pe - p < 1)
- return LZO_MORE_DATA;
- p = Get8(p, &tmp, &adler32, &crc32);
- h->Method = tmp;
- if (h->Version >= 0x0940) {
- if (pe - p < 1)
- return LZO_MORE_DATA;
- p = Get8(p, &tmp, &adler32, &crc32);
- h->Level = tmp;
- }
-
- // .Flags
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->Flags, &adler32, &crc32);
-
- // .Filter
- if (h->Flags & LZO_H_FILTER) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->Filter, &adler32, &crc32);
- }
-
- // .Mode
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->Mode, &adler32, &crc32);
-
- // .MTimeLow
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->MTimeLow, &adler32, &crc32);
-
- // .MTimeHigh
- if (h->Version >= 0x0940) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->MTimeHigh, &adler32, &crc32);
- }
- if (h->Version < 0x0120) {
- if (h->MTimeLow == 0xffffffffUL) {
- h->MTimeLow = 0;
- }
- h->MTimeHigh = 0;
- }
-
- // .Name
- if (pe - p < 1)
- return LZO_MORE_DATA;
- p = Get8(p, &tmp, &adler32, &crc32);
- if (tmp > 0) {
- if (pe - p < tmp)
- return LZO_MORE_DATA;
- adler32 = lzo_adler32(adler32, p, tmp);
- crc32 = lzo_crc32(crc32, p, tmp);
-
- ::memcpy(h->Name, p, tmp);
- p += tmp;
- }
-
- if (h->Flags & LZO_H_CRC32) {
- checksum = crc32;
- } else {
- checksum = adler32;
- }
-
- // .HeaderChecksum
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->HeaderChecksum, &adler32, &crc32);
- if (h->HeaderChecksum != checksum) {
- return LZO_WRONG_CHECKSUM;
- }
-
- // XXX(sandello): This is internal Statbox constraint.
- // XXX(aozeritsky): Statbox uses Method = 2, Java uses Method = 1
- // XXX(aozeritsky): Both methods use the same decompression function
- if (!(h->Method == 1 || h->Method == 2)) {
- return LZO_WRONG_METHOD;
- }
-
- if (h->Flags & LZO_H_EXTRA_FIELD) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &h->ExtraFieldLen, &adler32, &crc32);
- if (pe - p < h->ExtraFieldLen)
- return LZO_MORE_DATA;
- p += h->ExtraFieldLen;
- }
-
- // OK
- InputOffset += p - pb;
- return LZO_OK;
-}
-
-int TLzopDecompress::TImpl::ConsumeData() {
- using namespace NPrivate::NLzop;
-
- THeader* h = &Header;
-
- ui32 adler32 = 1;
- ui32 crc32 = 0;
-
- ui32 dAdler32 = 1;
- ui32 dCrc32 = 0;
- ui32 cAdler32 = 1;
- ui32 cCrc32 = 0;
-
- ui32 dstLen;
- ui32 srcLen;
-
- unsigned char* p;
- unsigned char* pb;
- unsigned char* pe;
-
- if (h->Version == 0) {
- return ConsumeHeader();
- }
-
- pb = p = (unsigned char*)InputData.Data() + InputOffset;
- pe = (unsigned char*)InputData.Pos();
-
- // dstLen
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &dstLen, &adler32, &crc32);
-
- if (dstLen == 0) {
- InputOffset += p - pb;
- return LZO_END_OF_STREAM;
- }
- if (dstLen == 0xffffffffUL) {
- return LZO_WRONG_DST_LEN;
- }
- if (dstLen > 64 * 1024 * 1024) {
- return LZO_DST_LEN_TOO_BIG;
- }
-
- // srcLen
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &srcLen, &adler32, &crc32);
-
- if (srcLen <= 0 || srcLen > dstLen) {
- return LZO_WRONG_SRC_LEN;
- }
-
- if (h->Flags & LZO_ADLER32_D) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &dAdler32, &adler32, &crc32);
- }
- if (h->Flags & LZO_CRC32_D) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &dCrc32, &adler32, &crc32);
- }
-
- if (h->Flags & LZO_ADLER32_C) {
- if (srcLen < dstLen) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &cAdler32, &adler32, &crc32);
- } else {
- if (!(h->Flags & LZO_ADLER32_D))
- ythrow yexception() << "h->Flags & LZO_ADLER32_C & ~LZO_ADLER32_D";
- cAdler32 = dAdler32;
- }
- }
- if (h->Flags & LZO_CRC32_C) {
- if (srcLen < dstLen) {
- if (pe - p < 4)
- return LZO_MORE_DATA;
- p = Get32(p, &cCrc32, &adler32, &crc32);
- } else {
- if (!(h->Flags & LZO_CRC32_D))
- ythrow yexception() << "h->Flags & LZO_CRC32_C & ~LZO_CRC32_D";
- cCrc32 = dCrc32;
- }
- }
-
- // Rock'n'roll! Check'n'consume!
- if (pe - p < srcLen) {
- Hint = (p - pb) + srcLen;
- return LZO_MORE_DATA;
- }
-
- if (h->Flags & LZO_ADLER32_C) {
- ui32 checksum;
- checksum = lzo_adler32(1, p, srcLen);
- if (checksum != cAdler32) {
- return LZO_INVALID_SRC_ADLER32;
- }
- }
- if (h->Flags & LZO_CRC32_C) {
- ui32 checksum;
- checksum = lzo_crc32(1, p, srcLen);
- if (checksum != cCrc32) {
- return LZO_INVALID_SRC_CRC32;
- }
- }
-
- if (OutputData.Avail() < dstLen) {
- OutputData.Reserve(RoundUpToPow2(2 * (OutputData.Size() + dstLen)));
- }
-
- unsigned char* output = (unsigned char*)OutputData.Pos();
- OutputData.Advance(dstLen);
-
- if (srcLen < dstLen) {
- lzo_uint tmp;
- int rv;
-
- tmp = dstLen;
- rv = lzo1x_decompress_safe(
- p,
- srcLen,
- output,
- &tmp,
- 0);
-
- if (rv != LZO_E_OK || tmp != dstLen) {
- return LZO_DECOMPRESS_ERROR;
- }
- } else {
- if (!(dstLen == srcLen)) {
- ythrow yexception() << "dstLen == srcLen";
- }
- ::memcpy(output, p, srcLen);
- }
-
- p += srcLen;
-
- // Check again.
- if (h->Flags & LZO_ADLER32_D) {
- ui32 checksum;
- checksum = lzo_adler32(1, output, dstLen);
- if (checksum != dAdler32) {
- return LZO_INVALID_DST_ADLER32;
- }
- }
- if (h->Flags & LZO_CRC32_D) {
- ui32 checksum;
- checksum = lzo_crc32(1, output, dstLen);
- if (checksum != dCrc32) {
- return LZO_INVALID_DST_CRC32;
- }
- }
-
- // OK
- InputOffset += p - pb;
- return LZO_OK;
-}
-
-TLzopDecompress::TLzopDecompress(IInputStream* slave, ui32 initialBufferSize)
- : Impl_(new TImpl(slave, initialBufferSize))
-{
-}
-
-TLzopDecompress::~TLzopDecompress() {
-}
-
-size_t TLzopDecompress::DoRead(void* buffer, size_t length) {
- return Impl_->Read(buffer, length);
-}
diff --git a/library/cpp/streams/lzop/lzop.h b/library/cpp/streams/lzop/lzop.h
deleted file mode 100644
index 523e5ad01b..0000000000
--- a/library/cpp/streams/lzop/lzop.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/generic/yexception.h>
-#include <util/stream/input.h>
-#include <util/stream/output.h>
-
-class TLzopCompress: public IOutputStream {
-public:
- TLzopCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15);
- ~TLzopCompress() override;
-
-private:
- void DoWrite(const void* buf, size_t len) override;
- void DoFlush() override;
- void DoFinish() override;
-
-private:
- class TImpl;
- THolder<TImpl> Impl_;
-};
-
-class TLzopDecompress: public IInputStream {
-public:
- TLzopDecompress(IInputStream* slave, ui32 initialBufferSize = 1 << 16);
- ~TLzopDecompress() override;
-
-private:
- size_t DoRead(void* buf, size_t len) override;
-
-private:
- class TImpl;
- THolder<TImpl> Impl_;
-};
diff --git a/library/cpp/streams/lzop/lzop_ut.cpp b/library/cpp/streams/lzop/lzop_ut.cpp
deleted file mode 100644
index 38b35b1c37..0000000000
--- a/library/cpp/streams/lzop/lzop_ut.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-#include "lzop.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/tempfile.h>
-#include <util/stream/file.h>
-
-#define ZDATA "./lzop_data"
-
-Y_UNIT_TEST_SUITE(TLzopTest) {
- static const TString data = "8s7d5vc6s5vc67sa4c65ascx6asd4xcv76adsfxv76s";
- static const TString data2 = "cn8wk2bd9vb3vdfif83g1ks94bfiovtwv";
-
- Y_UNIT_TEST(Compress) {
- TUnbufferedFileOutput o(ZDATA);
- TLzopCompress c(&o);
-
- c.Write(data.data(), data.size());
- c.Finish();
- o.Finish();
- }
-
- Y_UNIT_TEST(Decompress) {
- TTempFile tmpFile(ZDATA);
-
- {
- TUnbufferedFileInput i(ZDATA);
- TLzopDecompress d(&i);
-
- UNIT_ASSERT_EQUAL(d.ReadLine(), data);
- }
- }
-
- Y_UNIT_TEST(DecompressTwoStreams) {
- // Check that Decompress(Compress(X) + Compress(Y)) == X + Y
- TTempFile tmpFile(ZDATA);
- {
- TUnbufferedFileOutput o(ZDATA);
- TLzopCompress c1(&o);
- c1.Write(data.data(), data.size());
- c1.Finish();
- TLzopCompress c2(&o);
- c2.Write(data2.data(), data2.size());
- c2.Finish();
- o.Finish();
- }
- {
- TUnbufferedFileInput i(ZDATA);
- TLzopDecompress d(&i);
-
- UNIT_ASSERT_EQUAL(d.ReadLine(), data + data2);
- }
- }
-}
diff --git a/library/cpp/string_utils/secret_string/CMakeLists.txt b/library/cpp/string_utils/secret_string/CMakeLists.txt
deleted file mode 100644
index e801dca812..0000000000
--- a/library/cpp/string_utils/secret_string/CMakeLists.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-string_utils-secret_string)
-target_link_libraries(cpp-string_utils-secret_string PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-string_utils-ztstrbuf
-)
-target_sources(cpp-string_utils-secret_string PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/string_utils/secret_string/secret_string.cpp
-)
diff --git a/library/cpp/string_utils/secret_string/secret_string.cpp b/library/cpp/string_utils/secret_string/secret_string.cpp
deleted file mode 100644
index 3b68d3cd27..0000000000
--- a/library/cpp/string_utils/secret_string/secret_string.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-#include "secret_string.h"
-
-#include <util/system/madvise.h>
-
-namespace NSecretString {
- TSecretString::TSecretString(TStringBuf value) {
- Init(value);
- }
-
- TSecretString::~TSecretString() {
- try {
- Clear();
- } catch (...) {
- }
- }
-
- TSecretString& TSecretString::operator=(const TSecretString& o) {
- if (&o == this) {
- return *this;
- }
-
- Init(o.Value_);
-
- return *this;
- }
-
- /**
- * It is not honest "move". Actually it is copy-assignment with cleaning of other instance.
- * This way allowes to avoid side effects of string optimizations:
- * Copy-On-Write or Short-String-Optimization
- */
- TSecretString& TSecretString::operator=(TSecretString&& o) {
- if (&o == this) {
- return *this;
- }
-
- Init(o.Value_);
- o.Clear();
-
- return *this;
- }
-
- TSecretString& TSecretString::operator=(const TStringBuf o) {
- Init(o);
-
- return *this;
- }
-
- void TSecretString::Init(TStringBuf value) {
- Clear();
- if (value.empty()) {
- return;
- }
-
- Value_ = value;
- MadviseExcludeFromCoreDump(Value_);
- }
-
- void TSecretString::Clear() {
- if (Value_.empty()) {
- return;
- }
-
- SecureZero((void*)Value_.data(), Value_.size());
- MadviseIncludeIntoCoreDump(Value_);
- Value_.clear();
- }
-}
diff --git a/library/cpp/string_utils/secret_string/secret_string.h b/library/cpp/string_utils/secret_string/secret_string.h
deleted file mode 100644
index fdb9f6a85c..0000000000
--- a/library/cpp/string_utils/secret_string/secret_string.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#pragma once
-
-#include <library/cpp/string_utils/ztstrbuf/ztstrbuf.h>
-
-#include <util/generic/string.h>
-
-namespace NSecretString {
- /**
- * TSecretString allowes to store some long lived secrets in "secure" storage in memory.
- * Common usage:
- * 1) read secret value from disk/env/etc
- * 2) put it into TSecretString
- * 3) destory secret copy from 1)
- *
- * Useful scenerios for TSecretString:
- * - in memory only tasks: using key to create crypto signature;
- * - rare network cases: db password on connection or OAuth token in background tasks.
- * These cases disclosure the secret
- * because of sending it over network with some I/O frameworks.
- * Usually such frameworks copy input params to provide network protocol: gRPC, for example.
- *
- * Supported features:
- * 1. Exclude secret from core dump.
- * madvise(MADV_DONTDUMP) in ctor excludes full memory page from core dump.
- * madvise(MADV_DODUMP) in dtor reverts previous action.
- * 2. Zero memory before free.
- *
- * Code dump looks like this:
-(gdb) print s
-$1 = (const TSecretString &) @0x7fff23c4c560: {
- Value_ = {<TStringBase<TBasicString<char, std::__y1::char_traits<char> >, char, std::__y1::char_traits<char> >> = {
- static npos = <optimized out>}, Data_ = 0x107c001d8 <error: Cannot access memory at address 0x107c001d8>}}
- */
-
- class TSecretString {
- public:
- TSecretString() = default;
- TSecretString(TStringBuf value);
- ~TSecretString();
-
- TSecretString(const TSecretString& o)
- : TSecretString(o.Value())
- {
- }
-
- TSecretString(TSecretString&& o)
- : TSecretString(o.Value())
- {
- o.Clear();
- }
-
- TSecretString& operator=(const TSecretString& o);
- TSecretString& operator=(TSecretString&& o);
-
- TSecretString& operator=(const TStringBuf o);
-
- operator TZtStringBuf() const {
- return Value();
- }
-
- // Provides zero terminated string
- TZtStringBuf Value() const {
- return TZtStringBuf(Value_);
- }
-
- private:
- // TStringBuf breaks Copy-On-Write to provide correct copy-ctor and copy-assignment
- void Init(TStringBuf value);
- void Clear();
-
- private:
- TString Value_;
- };
-}
diff --git a/library/cpp/string_utils/secret_string/ut/secret_string_ut.cpp b/library/cpp/string_utils/secret_string/ut/secret_string_ut.cpp
deleted file mode 100644
index 681b75368f..0000000000
--- a/library/cpp/string_utils/secret_string/ut/secret_string_ut.cpp
+++ /dev/null
@@ -1,147 +0,0 @@
-#include <library/cpp/string_utils/secret_string/secret_string.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NSecretString;
-
-Y_UNIT_TEST_SUITE(SecretTest) {
- Y_UNIT_TEST(Common) {
- TSecretString s;
- UNIT_ASSERT_VALUES_EQUAL("", s.Value());
- UNIT_ASSERT_VALUES_EQUAL("", (TStringBuf)s);
-
- TSecretString s2("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", (TStringBuf)s2);
- }
-
- Y_UNIT_TEST(CopyCtor1) {
- TSecretString s1("qwerty");
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
-
- {
- TSecretString s2(s1);
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
- }
-
- Y_UNIT_TEST(CopyCtor2) {
- auto s1 = MakeHolder<TSecretString>("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
-
- TSecretString s2(*s1);
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- s1.Reset();
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- Y_UNIT_TEST(MoveCtor1) {
- TSecretString s1("qwerty");
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
-
- {
- TSecretString s2(std::move(s1));
- UNIT_ASSERT_VALUES_EQUAL("", s1.Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- UNIT_ASSERT_VALUES_EQUAL("", s1.Value());
- }
-
- Y_UNIT_TEST(MoveCtor2) {
- auto s1 = MakeHolder<TSecretString>("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
-
- TSecretString s2(std::move(*s1));
- UNIT_ASSERT_VALUES_EQUAL("", s1->Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- s1.Reset();
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- Y_UNIT_TEST(CopyAssignment1) {
- TSecretString s1("qwerty");
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
-
- {
- TSecretString s2;
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
-
- s2 = s1;
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
- }
-
- Y_UNIT_TEST(CopyAssignment2) {
- auto s1 = MakeHolder<TSecretString>("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
-
- TSecretString s2;
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
-
- s2 = *s1;
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- s1.Reset();
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- TSecretString s3;
- s2 = s3;
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
- }
-
- Y_UNIT_TEST(MoveAssignment1) {
- TSecretString s1("qwerty");
-
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1.Value());
-
- {
- TSecretString s2;
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
-
- s2 = std::move(s1);
- UNIT_ASSERT_VALUES_EQUAL("", s1.Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
- }
-
- UNIT_ASSERT_VALUES_EQUAL("", s1.Value());
- }
-
- Y_UNIT_TEST(MoveAssignment2) {
- auto s1 = MakeHolder<TSecretString>("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s1->Value());
-
- TSecretString s2;
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
-
- s2 = std::move(*s1);
- UNIT_ASSERT_VALUES_EQUAL("", s1->Value());
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- s1.Reset();
- UNIT_ASSERT_VALUES_EQUAL("qwerty", s2.Value());
-
- TSecretString s3;
- s2 = std::move(s3);
- UNIT_ASSERT_VALUES_EQUAL("", s2.Value());
- }
-
- Y_UNIT_TEST(ZeroTerminated) {
- TSecretString s("qwerty");
-
- UNIT_ASSERT_VALUES_EQUAL(s.Value().size(), strlen(s.Value().data()));
- }
-}
diff --git a/library/cpp/threading/atomic/CMakeLists.txt b/library/cpp/threading/atomic/CMakeLists.txt
deleted file mode 100644
index 782b274948..0000000000
--- a/library/cpp/threading/atomic/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-threading-atomic)
-target_link_libraries(cpp-threading-atomic PUBLIC
- contrib-libs-cxxsupp
- yutil
-)
-target_sources(cpp-threading-atomic PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/threading/atomic/bool.cpp
-)
diff --git a/library/cpp/tvmauth/CMakeLists.txt b/library/cpp/tvmauth/CMakeLists.txt
deleted file mode 100644
index e0ac60da79..0000000000
--- a/library/cpp/tvmauth/CMakeLists.txt
+++ /dev/null
@@ -1,64 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(library-cpp-tvmauth)
-target_link_libraries(library-cpp-tvmauth PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-string_utils-secret_string
- tvmauth-src-protos
- ticket_parser
- tools-enum_parser-enum_serialization_runtime
- library-cpp-resource
-)
-target_sources(library-cpp-tvmauth PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/deprecated/service_context.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/deprecated/user_context.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/parser.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/service_impl.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/service_ticket.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/status.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/unittest.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/user_impl.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/user_ticket.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/utils.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/version.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/utils.cpp
-)
-generate_enum_serilization(library-cpp-tvmauth
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/checked_user_ticket.h
- INCLUDE_HEADERS
- library/cpp/tvmauth/checked_user_ticket.h
-)
-generate_enum_serilization(library-cpp-tvmauth
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/ticket_status.h
- INCLUDE_HEADERS
- library/cpp/tvmauth/ticket_status.h
-)
-
-add_global_library_for(library-cpp-tvmauth.global library-cpp-tvmauth)
-target_link_libraries(library-cpp-tvmauth.global PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-string_utils-secret_string
- tvmauth-src-protos
- ticket_parser
- tools-enum_parser-enum_serialization_runtime
- library-cpp-resource
-)
-target_sources(library-cpp-tvmauth.global PRIVATE
- ${CMAKE_BINARY_DIR}/library/cpp/tvmauth/0490e525f24df2be33c25916359673d0.cpp
-)
-resources(library-cpp-tvmauth.global
- ${CMAKE_BINARY_DIR}/library/cpp/tvmauth/0490e525f24df2be33c25916359673d0.cpp
- INPUTS
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/version
- KEYS
- /builtin/version
-)
diff --git a/library/cpp/tvmauth/README.md b/library/cpp/tvmauth/README.md
deleted file mode 100644
index ec64bbbcdb..0000000000
--- a/library/cpp/tvmauth/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-This part of library contains primitives for TVM operation.
-Please use high-level [TTvmClient](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/README.md).
diff --git a/library/cpp/tvmauth/a.yaml b/library/cpp/tvmauth/a.yaml
deleted file mode 100644
index 228ff3af1e..0000000000
--- a/library/cpp/tvmauth/a.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-service: passport_infra
-title: tvmauth (c++)
-
-arcanum:
- review:
- auto_assign: true
-
- groups:
- - name: backend-developers
- roles: developer
-
- rules:
- - reviewers:
- name: backend-developers
- ship: 2
- assign: 2
-
-ci:
- release-title-source: flow
- autocheck:
- fast-targets:
- - library/c/tvmauth
- - library/cpp/tvmauth
- - library/go/yandex/tvm
- - library/java/tvmauth
- - library/python/deprecated/ticket_parser2
- - library/python/tvmauth
- - passport
- strong: true
diff --git a/library/cpp/tvmauth/checked_service_ticket.h b/library/cpp/tvmauth/checked_service_ticket.h
deleted file mode 100644
index cf4c5c43e3..0000000000
--- a/library/cpp/tvmauth/checked_service_ticket.h
+++ /dev/null
@@ -1,71 +0,0 @@
-#pragma once
-
-#include "ticket_status.h"
-#include "type.h"
-#include "utils.h"
-
-#include <util/generic/ptr.h>
-
-namespace NTvmAuth::NInternal {
- class TCanningKnife;
-}
-
-namespace NTvmAuth {
- class TCheckedServiceTicket {
- public:
- class TImpl;
-
- TCheckedServiceTicket(THolder<TImpl> impl);
- TCheckedServiceTicket(TCheckedServiceTicket&& o);
- ~TCheckedServiceTicket();
-
- TCheckedServiceTicket& operator=(TCheckedServiceTicket&&);
-
- /*!
- * @return True value if ticket parsed and checked successfully
- */
- explicit operator bool() const;
-
- /*!
- * You should check src with your ACL
- * @return TvmId of request source
- */
- TTvmId GetSrc() const;
-
- /*!
- * @return Ticket check status
- */
- ETicketStatus GetStatus() const;
-
- /*!
- * DebugInfo is human readable data for debug purposes
- * @return Serialized ticket
- */
- TString DebugInfo() const;
-
- /*!
- * IssuerUID is UID of developer who is debuging something,
- * so he(she) issued ServiceTicket with his(her) ssh-sign:
- * it is grant_type=sshkey in tvm-api.
- * https://wiki.yandex-team.ru/passport/tvm2/debug/#sxoditvapizakrytoeserviceticketami
- * @return uid
- */
- TMaybe<TUid> GetIssuerUid() const;
-
- public: // for python binding
- TCheckedServiceTicket() = default;
-
- private:
- THolder<TImpl> Impl_;
- friend class NInternal::TCanningKnife;
- };
-
- namespace NBlackboxTvmId {
- const TStringBuf Prod = "222";
- const TStringBuf Test = "224";
- const TStringBuf ProdYateam = "223";
- const TStringBuf TestYateam = "225";
- const TStringBuf Stress = "226";
- const TStringBuf Mimino = "239";
- }
-}
diff --git a/library/cpp/tvmauth/checked_user_ticket.h b/library/cpp/tvmauth/checked_user_ticket.h
deleted file mode 100644
index 16a2a6dc30..0000000000
--- a/library/cpp/tvmauth/checked_user_ticket.h
+++ /dev/null
@@ -1,91 +0,0 @@
-#pragma once
-
-#include "ticket_status.h"
-#include "type.h"
-#include "utils.h"
-
-#include <util/generic/ptr.h>
-
-namespace NTvmAuth::NInternal {
- class TCanningKnife;
-}
-
-namespace NTvmAuth {
- /*!
- * BlackboxEnv describes environment of Passport:
- * https://wiki.yandex-team.ru/passport/tvm2/user-ticket/#0-opredeljaemsjasokruzhenijami
- */
- enum class EBlackboxEnv: ui8 {
- Prod,
- Test,
- ProdYateam,
- TestYateam,
- Stress
- };
-
- /*!
- * UserTicket contains only valid users.
- * Details: https://wiki.yandex-team.ru/passport/tvm2/user-ticket/#chtoestvusertickete
- */
- class TCheckedUserTicket {
- public:
- class TImpl;
-
- TCheckedUserTicket(THolder<TImpl> impl);
- TCheckedUserTicket(TCheckedUserTicket&&);
- ~TCheckedUserTicket();
-
- TCheckedUserTicket& operator=(TCheckedUserTicket&&);
-
- /*!
- * @return True value if ticket parsed and checked successfully
- */
- explicit operator bool() const;
-
- /*!
- * Never empty
- * @return UIDs of users listed in ticket
- */
- const TUids& GetUids() const;
-
- /*!
- * Maybe 0
- * @return Default user in ticket
- */
- TUid GetDefaultUid() const;
-
- /*!
- * Scopes inherited from credential - never empty
- * @return Newly constructed vector of scopes
- */
- const TScopes& GetScopes() const;
-
- /*!
- * Check if scope presented in ticket
- */
- bool HasScope(TStringBuf scopeName) const;
-
- /*!
- * @return Ticket check status
- */
- ETicketStatus GetStatus() const;
-
- /*!
- * DebugInfo is human readable data for debug purposes
- * @return Serialized ticket
- */
- TString DebugInfo() const;
-
- /*!
- * Env of user
- */
- EBlackboxEnv GetEnv() const;
-
- public: // for python binding
- TCheckedUserTicket() = default;
-
- private:
- THolder<TImpl> Impl_;
- friend class NInternal::TCanningKnife;
- };
-}
diff --git a/library/cpp/tvmauth/client/CMakeLists.txt b/library/cpp/tvmauth/client/CMakeLists.txt
deleted file mode 100644
index 92762a80ee..0000000000
--- a/library/cpp/tvmauth/client/CMakeLists.txt
+++ /dev/null
@@ -1,61 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(cpp-tvmauth-client)
-target_link_libraries(cpp-tvmauth-client PUBLIC
- contrib-libs-cxxsupp
- yutil
- cpp-http-simple
- library-cpp-json
- cpp-openssl-crypto
- cpp-streams-brotli
- cpp-streams-zstd
- cpp-string_utils-quote
- library-cpp-tvmauth
- misc-retry_settings-v1
- tools-enum_parser-enum_serialization_runtime
-)
-target_sources(cpp-tvmauth-client PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/client_status.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/facade.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/logger.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/api/roles_fetcher.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/api/settings.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/api/threaded_updater.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/async_updater.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/disk_cache.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/last_error.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/proc_info.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/roles/decoder.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/roles/entities_index.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/roles/parser.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/roles/roles.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/threaded_updater.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/tool/meta_info.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/tool/roles_fetcher.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/tool/settings.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/tool/threaded_updater.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/utils.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/mocked_updater.cpp
-)
-generate_enum_serilization(cpp-tvmauth-client
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/client_status.h
- INCLUDE_HEADERS
- library/cpp/tvmauth/client/client_status.h
-)
-generate_enum_serilization(cpp-tvmauth-client
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/async_updater.h
- INCLUDE_HEADERS
- library/cpp/tvmauth/client/misc/async_updater.h
-)
-generate_enum_serilization(cpp-tvmauth-client
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/last_error.h
- INCLUDE_HEADERS
- library/cpp/tvmauth/client/misc/last_error.h
-)
diff --git a/library/cpp/tvmauth/client/README.md b/library/cpp/tvmauth/client/README.md
deleted file mode 100644
index cda6a22d3c..0000000000
--- a/library/cpp/tvmauth/client/README.md
+++ /dev/null
@@ -1,84 +0,0 @@
-Overview
-===
-This library provides ability to operate with TVM. Library is fast enough to get or check tickets for every request without burning CPU.
-
-[Home page of project](https://wiki.yandex-team.ru/passport/tvm2/)
-You can find some examples in [here](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/examples).
-
-You can ask questions: [PASSPORTDUTY](https://st.yandex-team.ru/createTicket?queue=PASSPORTDUTY&_form=77618)
-
-TvmClient
-===
-Don't forget to collect logs from client.
-___
-`TvmClient` allowes:
-1. `GetServiceTicketFor()` - to fetch ServiceTicket for outgoing request
-2. `CheckServiceTicket()` - to check ServiceTicket from incoming request
-3. `CheckUserTicket()` - to check UserTicket from incoming request
-4. `GetRoles()` - to get roles from IDM
-
-All methods are thread-safe.
-
-You should check status of `CheckedServiceTicket` or `CheckedUserTicket` for equality 'Ok'. You can get ticket fields (src/uids/scopes) only for correct ticket. Otherwise exception will be thrown.
-___
-You should check status of client with `GetStatus()`:
-* `OK` - nothing to do here
-* `Warning` - **you should trigger your monitoring alert**
-
- Normal operation of TvmClient is still possible but there are problems with refreshing cache, so it is expiring.
- Is tvm-api.yandex.net accessible?
- Have you changed your TVM-secret or your backend (dst) deleted its TVM-client?
-
-* `Error` - **you should trigger your monitoring alert and close this instance for user-traffic**
-
- TvmClient's cache is already invalid (expired) or soon will be: you can't check valid ServiceTicket or be authenticated by your backends (dsts)
-
-___
-Constructor creates system thread for refreshing cache - so do not fork your proccess after creating `TTvmClient` instance. Constructor leads to network I/O. Other methods always use memory.
-
-Exceptions maybe thrown from constructor:
-* `TRetriableException` - maybe some network trouble: you can try to create client one more time.
-* `TNonRetriableException` - settings are bad: fix them.
-___
-You can choose way for fetching data for your service operation:
-* http://localhost:{port}/tvm - recomended way
-* https://tvm-api.yandex.net
-
-TvmTool
-------------
-`TTvmClient` uses local http-interface to get state. This interface can be provided with tvmtool (local daemon) or Qloud/YP (local http api in container).
-See more: https://wiki.yandex-team.ru/passport/tvm2/tvm-daemon/.
-
-`TTvmClient` fetches configuration from tvmtool, so you need only to tell client how to connect to it and tell which alias of tvm id should be used for this `TvmClient` instance.
-
-TvmApi
-------------
-First of all: please use `DiskCacheDir` - it provides reliability for your service and for tvm-api.
-Please check restrictions of this field.
-
-Roles
-===
-[Example](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/examples/create_with_tvmapi/create.cpp?rev=r8888584#L84)
-
-You need to configure roles fetching
-------------
-1. Enable disk cache: [DiskCacheDir](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/api/settings.h?rev=r9001419#L54)
-
-2. Enable ServiceTicket fetching:
- [SelfTvmId](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/api/settings.h?rev=r9001419#L57) + [Secret](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/api/settings.h?rev=r9001419#L60)
-3. Enable roles fetching from tirole:
- [FetchRolesForIdmSystemSlug](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/api/settings.h?rev=r9001419#L78)
-
-You need to use roles for request check
-------------
-1. Check ServiceTicket and/or UserTicket - as usual:
- [CheckServiceTicket()](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/facade.h?rev=r7890770#L91)/[CheckUserTicket()](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/facade.h?rev=r7890770#L99)
-
-2. Get actual roles from `TvmClient`: [GetRoles()](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/facade.h?rev=r7890770#L105)
-
-3. Use roles
- - case#1: [get](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/roles/roles.h?rev=r7890770#L37-46) role list for service or user and check for the exact role you need.
- - case#2: use [shortcuts](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/misc/roles/roles.h?rev=r7890770#L50) - they are wrappers for case#1
-
-4. If consumer (service or user) has required role, you can perform request.
- If consumer doesn't have required role, you should show error message with useful message.
diff --git a/library/cpp/tvmauth/client/client_status.cpp b/library/cpp/tvmauth/client/client_status.cpp
deleted file mode 100644
index eca35ba22b..0000000000
--- a/library/cpp/tvmauth/client/client_status.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-#include "client_status.h"
-
-template <>
-void Out<NTvmAuth::TClientStatus>(IOutputStream& out, const NTvmAuth::TClientStatus& s) {
- out << s.GetCode() << ": " << s.GetLastError();
-}
diff --git a/library/cpp/tvmauth/client/client_status.h b/library/cpp/tvmauth/client/client_status.h
deleted file mode 100644
index bbaf29d289..0000000000
--- a/library/cpp/tvmauth/client/client_status.h
+++ /dev/null
@@ -1,82 +0,0 @@
-#pragma once
-
-#include <util/generic/string.h>
-#include <util/string/builder.h>
-
-namespace NTvmAuth {
- class TClientStatus {
- public:
- enum ECode {
- Ok,
- Warning,
- Error,
- IncompleteTicketsSet,
- };
-
- TClientStatus(ECode state, TString&& lastError)
- : Code_(state)
- , LastError_(std::move(lastError))
- {
- }
-
- TClientStatus() = default;
- TClientStatus(const TClientStatus&) = default;
- TClientStatus(TClientStatus&&) = default;
-
- TClientStatus& operator=(const TClientStatus&) = default;
- TClientStatus& operator=(TClientStatus&&) = default;
-
- ECode GetCode() const {
- return Code_;
- }
-
- const TString& GetLastError() const {
- return LastError_;
- }
-
- TString CreateJugglerMessage() const {
- return TStringBuilder() << GetJugglerCode() << ";TvmClient: " << LastError_ << "\n";
- }
-
- private:
- int32_t GetJugglerCode() const {
- switch (Code_) {
- case ECode::Ok:
- return 0; // OK juggler check state
- case ECode::Warning:
- case ECode::IncompleteTicketsSet:
- return 1; // WARN juggler check state
- case ECode::Error:
- return 2; // CRIT juggler check state
- }
- return 2; // This should not happen, so set check state as CRIT.
- }
-
- ECode Code_ = Ok;
- TString LastError_;
- };
-
- static inline bool operator==(const TClientStatus& l, const TClientStatus& r) noexcept {
- return l.GetCode() == r.GetCode() && l.GetLastError() == r.GetLastError();
- }
-
- static inline bool operator==(const TClientStatus& l, const TClientStatus::ECode r) noexcept {
- return l.GetCode() == r;
- }
-
- static inline bool operator==(const TClientStatus::ECode l, const TClientStatus& r) noexcept {
- return r.GetCode() == l;
- }
-
- static inline bool operator!=(const TClientStatus& l, const TClientStatus& r) noexcept {
- return !(l == r);
- }
-
- static inline bool operator!=(const TClientStatus& l, const TClientStatus::ECode r) noexcept {
- return !(l == r);
- }
-
- static inline bool operator!=(const TClientStatus::ECode l, const TClientStatus& r) noexcept {
- return !(l == r);
- }
-}
diff --git a/library/cpp/tvmauth/client/examples/create_with_tvmapi/create.cpp b/library/cpp/tvmauth/client/examples/create_with_tvmapi/create.cpp
deleted file mode 100644
index c03a7a032f..0000000000
--- a/library/cpp/tvmauth/client/examples/create_with_tvmapi/create.cpp
+++ /dev/null
@@ -1,102 +0,0 @@
-#include <library/cpp/tvmauth/client/facade.h>
-
-namespace NExample {
- NTvmAuth::TTvmClient CreateClientForCheckingAllTicketsAndFetchingServiceTickets() {
- NTvmAuth::NTvmApi::TClientSettings setts{
- .DiskCacheDir = "/var/cache/my_service/tvm/",
- .SelfTvmId = 11,
- .Secret = (TStringBuf) "AAAAAAAAAAAAAAAAAAAAAA",
- .FetchServiceTicketsForDstsWithAliases = {
- {"bb", 224},
- {"datasync", 2000060},
- },
- .CheckServiceTickets = true,
- .CheckUserTicketsWithBbEnv = NTvmAuth::EBlackboxEnv::Test,
- };
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- // c.CheckServiceTicket("some service ticket")
- // c.CheckUserTicket("some user ticket")
- // c.GetServiceTicketFor("bb")
- // c.GetServiceTicketFor(224)
-
- return c;
- }
-
- NTvmAuth::TTvmClient CreateClientForCheckingAllTickets() {
- NTvmAuth::NTvmApi::TClientSettings setts{
- .DiskCacheDir = "/var/cache/my_service/tvm/",
- .SelfTvmId = 11,
- .CheckServiceTickets = true,
- .CheckUserTicketsWithBbEnv = NTvmAuth::EBlackboxEnv::Test,
- };
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- // c.CheckServiceTicket("some service ticket")
- // c.CheckUserTicket("some user ticket")
-
- return c;
- }
-
- NTvmAuth::TTvmClient CreateClientForFetchingServiceTickets() {
- NTvmAuth::NTvmApi::TClientSettings setts{
- .DiskCacheDir = "/var/cache/my_service/tvm/",
- .SelfTvmId = 11,
- .Secret = (TStringBuf) "AAAAAAAAAAAAAAAAAAAAAA",
- .FetchServiceTicketsForDstsWithAliases = {
- {"bb", 224},
- {"datasync", 2000060},
- },
- };
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- // c.GetServiceTicketFor("bb")
- // c.GetServiceTicketFor(224)
-
- return c;
- }
-
- NTvmAuth::TTvmClient CreateClientForCheckingServiceTickets() {
- NTvmAuth::NTvmApi::TClientSettings setts{
- .DiskCacheDir = "/var/cache/my_service/tvm/",
- .SelfTvmId = 11,
- .CheckServiceTickets = true,
- };
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- // c.CheckServiceTicket("some service ticket")
-
- return c;
- }
-
- NTvmAuth::TTvmClient CreateClientForCheckingServiceTicketsWithRoles() {
- NTvmAuth::NTvmApi::TClientSettings setts{
- .DiskCacheDir = "/var/cache/my_service/tvm/",
- .SelfTvmId = 11,
- .Secret = (TStringBuf) "AAAAAAAAAAAAAAAAAAAAAA",
- .CheckServiceTickets = true,
- .FetchRolesForIdmSystemSlug = "passporttestservice",
- };
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- // auto t = c.CheckServiceTicket("some service ticket")
- // c.GetRoles()->CheckServiceRole(t, "some role");
-
- return c;
- }
-}
diff --git a/library/cpp/tvmauth/client/examples/create_with_tvmtool/create.cpp b/library/cpp/tvmauth/client/examples/create_with_tvmtool/create.cpp
deleted file mode 100644
index a87d3e705d..0000000000
--- a/library/cpp/tvmauth/client/examples/create_with_tvmtool/create.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-#include <library/cpp/tvmauth/client/facade.h>
-
-namespace NExample {
- // Possibility of using functions depends on config of tvmtool
- // CheckServiceTicket
- // CheckUserTicket
- // GetServiceTicketFor
-
- NTvmAuth::TTvmClient CreateClientInQloudOrYandexDeploy() {
- NTvmAuth::NTvmTool::TClientSettings setts(
- "my_service" // specified in Qloud/YP/tvmtool interface
- );
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- return c;
- }
-
- NTvmAuth::TTvmClient CreateClientForDevOrTests() {
- NTvmAuth::NTvmTool::TClientSettings setts(
- "my_service" // specified in Qloud/YP/tvmtool interface
- );
- setts.SetPort(18080);
- setts.SetAuthToken("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- NTvmAuth::TTvmClient c(setts, log);
-
- return c;
- }
-}
diff --git a/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.cpp b/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.cpp
deleted file mode 100644
index 075bf0bded..0000000000
--- a/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.cpp
+++ /dev/null
@@ -1,84 +0,0 @@
-#include "service.h"
-
-#include <library/cpp/tvmauth/client/facade.h>
-
-#include <library/cpp/cgiparam/cgiparam.h>
-#include <library/cpp/http/server/response.h>
-#include <library/cpp/http/simple/http_client.h>
-#include <library/cpp/json/json_reader.h>
-
-namespace NExample {
- static const TString BACK_C = "BACK_C";
-
- TSomeService::TSomeService(const TConfig& cfg)
- : Config_(cfg)
- {
- NTvmAuth::TLoggerPtr log = MakeIntrusive<NTvmAuth::TCerrLogger>(7);
-
- Tvm_ = MakeHolder<NTvmAuth::TTvmClient>(
- NTvmAuth::NTvmTool::TClientSettings(
- "my_service" // specified in Qloud/YP/tvmtool interface
- ),
- log);
- }
-
- TSomeService::~TSomeService() {
- }
-
- void TSomeService::HandleRequest(THttpInput& in, THttpOutput& out) {
- auto servIt = std::find_if(in.Headers().Begin(),
- in.Headers().End(),
- [](const auto& h) { return h.Name() == "X-Ya-Service-Ticket"; });
- auto userIt = std::find_if(in.Headers().Begin(),
- in.Headers().End(),
- [](const auto& h) { return h.Name() == "X-Ya-User-Ticket"; });
- try {
- if (servIt == in.Headers().End() || userIt == in.Headers().End()) {
- ythrow yexception() << "Need tickets";
- }
-
- // WARNING: См. Здесь
- NTvmAuth::TCheckedServiceTicket st = Tvm_->CheckServiceTicket(servIt->Value());
- NTvmAuth::TCheckedUserTicket ut = Tvm_->CheckUserTicket(userIt->Value());
- if (!st || !ut) {
- ythrow yexception() << "Invalid tickets";
- }
-
- // WARNING: См. Здесь
- // Ждём ABC - после их релиза эти три строки можно будет удалить
- if (Config_.AllowedTvmIds.find(st.GetSrc()) == Config_.AllowedTvmIds.end()) {
- ythrow yexception() << "Consumer is not allowed";
- }
-
- // WARNING: См. Здесь
- if (!ut.HasScope("some_service:allow_secret_data")) {
- ythrow yexception() << "UserTicket does not have scopes for secret data";
- }
-
- // Access-log
- Cout << "Data fetched for: " << ut.GetDefaultUid() << Endl;
-
- THttpResponse resp(HTTP_OK);
- resp.SetContent(GetDataFromBackendC(userIt->Value()), "text/plain");
- resp.OutTo(out);
- } catch (...) {
- THttpResponse resp(HTTP_BAD_REQUEST);
- resp.SetContent("Request can not be performed", "text/plain");
- resp.OutTo(out);
- }
-
- out.Finish();
- }
-
- TString TSomeService::GetDataFromBackendC(const TString& userTicket) {
- TSimpleHttpClient cl("my_backend", // specified in Qloud/YP/tvmtool interface
- 80);
- TStringStream s;
- cl.DoGet("/api?",
- &s,
- // WARNING: См. Здесь
- {{"X-Ya-Service-Ticket", Tvm_->GetServiceTicketFor(BACK_C)},
- {"X-Ya-User-Ticket", userTicket}});
- return s.Str();
- }
-}
diff --git a/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.h b/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.h
deleted file mode 100644
index 8ff948334e..0000000000
--- a/library/cpp/tvmauth/client/examples/service_using_tvmtool_client/service.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include <library/cpp/http/io/stream.h>
-
-#include <util/generic/ptr.h>
-
-#include <unordered_set>
-
-namespace NTvmAuth {
- class TTvmClient;
-}
-
-namespace NExample {
- struct TConfig {
- using TAllowedTvmIds = std::unordered_set<ui32>;
-
- TAllowedTvmIds AllowedTvmIds;
- };
-
- class TSomeService {
- public:
- TSomeService(const TConfig& cfg);
- ~TSomeService();
-
- void HandleRequest(THttpInput& in, THttpOutput& out);
-
- private:
- TString GetDataFromBackendC(const TString& userTicket);
-
- private:
- // WARNING: См. Здесь
- TConfig Config_;
- THolder<NTvmAuth::TTvmClient> Tvm_;
- };
-}
diff --git a/library/cpp/tvmauth/client/exception.h b/library/cpp/tvmauth/client/exception.h
deleted file mode 100644
index 7639467671..0000000000
--- a/library/cpp/tvmauth/client/exception.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/exception.h>
-
-namespace NTvmAuth {
- class TClientException: public TTvmException {
- };
-
- class TRetriableException: public TClientException {
- };
- class TNonRetriableException: public TClientException {
- };
-
- class TIllegalUsage: public TNonRetriableException {
- };
-
- class TBrokenTvmClientSettings: public TIllegalUsage {
- };
- class TMissingServiceTicket: public TNonRetriableException {
- };
- class TPermissionDenied: public TNonRetriableException {
- };
-}
diff --git a/library/cpp/tvmauth/client/facade.cpp b/library/cpp/tvmauth/client/facade.cpp
deleted file mode 100644
index 6e77569441..0000000000
--- a/library/cpp/tvmauth/client/facade.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-#include "facade.h"
-
-#include "misc/checker.h"
-#include "misc/default_uid_checker.h"
-#include "misc/getter.h"
-#include "misc/src_checker.h"
-#include "misc/api/threaded_updater.h"
-#include "misc/tool/threaded_updater.h"
-
-namespace NTvmAuth {
- TTvmClient::TTvmClient(const NTvmTool::TClientSettings& settings, TLoggerPtr logger)
- : Updater_(NTvmTool::TThreadedUpdater::Create(settings, std::move(logger)))
- , Service_(MakeHolder<TServiceTicketChecker>(Updater_))
- , User_(MakeHolder<TUserTicketChecker>(Updater_))
- {
- if (Updater_->GetCachedServiceTickets()) {
- Tickets_ = MakeHolder<TServiceTicketGetter>(Updater_);
- }
- }
-
- TTvmClient::TTvmClient(const NTvmApi::TClientSettings& settings, TLoggerPtr logger)
- : Updater_(NTvmApi::TThreadedUpdater::Create(settings, std::move(logger)))
- {
- if (settings.IsServiceTicketFetchingRequired()) {
- Tickets_ = MakeHolder<TServiceTicketGetter>(Updater_);
- }
- if (settings.IsServiceTicketCheckingRequired()) {
- Service_ = MakeHolder<TServiceTicketChecker>(Updater_);
- }
- if (settings.IsUserTicketCheckingRequired()) {
- User_ = MakeHolder<TUserTicketChecker>(Updater_);
- }
- if (settings.IsRolesFetchingEnabled() && settings.ShouldCheckSrc) {
- SrcChecker_ = MakeHolder<TSrcChecker>(Updater_);
- }
- if (settings.IsRolesFetchingEnabled() && settings.ShouldCheckDefaultUid) {
- DefaultUidChecker_ = MakeHolder<TDefaultUidChecker>(Updater_);
- }
- }
-
- TTvmClient::TTvmClient(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- if (Updater_->GetCachedServiceTickets()) {
- Tickets_ = MakeHolder<TServiceTicketGetter>(Updater_);
- }
- if (Updater_->GetCachedServiceContext()) {
- Service_ = MakeHolder<TServiceTicketChecker>(Updater_);
- }
- if (Updater_->GetCachedUserContext()) {
- User_ = MakeHolder<TUserTicketChecker>(Updater_);
- }
-
- try {
- if (Updater_->GetRoles()) {
- SrcChecker_ = MakeHolder<TSrcChecker>(Updater_);
- DefaultUidChecker_ = MakeHolder<TDefaultUidChecker>(Updater_);
- }
- } catch (const TIllegalUsage&) {
- // it is a test probably
- }
- }
-
- TTvmClient::TTvmClient(TTvmClient&& o) = default;
- TTvmClient::~TTvmClient() = default;
- TTvmClient& TTvmClient::operator=(TTvmClient&& o) = default;
-
- TClientStatus TTvmClient::GetStatus() const {
- Y_ENSURE(Updater_);
- return Updater_->GetStatus();
- }
-
- TInstant TTvmClient::GetUpdateTimeOfPublicKeys() const {
- Y_ENSURE(Updater_);
- return Updater_->GetUpdateTimeOfPublicKeys();
- }
-
- TInstant TTvmClient::GetUpdateTimeOfServiceTickets() const {
- Y_ENSURE(Updater_);
- return Updater_->GetUpdateTimeOfServiceTickets();
- }
-
- TInstant TTvmClient::GetInvalidationTimeOfPublicKeys() const {
- Y_ENSURE(Updater_);
- return Updater_->GetInvalidationTimeOfPublicKeys();
- }
-
- TInstant TTvmClient::GetInvalidationTimeOfServiceTickets() const {
- Y_ENSURE(Updater_);
- return Updater_->GetInvalidationTimeOfServiceTickets();
- }
-
- TString TTvmClient::GetServiceTicketFor(const TClientSettings::TAlias& dst) const {
- Y_ENSURE_EX(Tickets_, TBrokenTvmClientSettings()
- << "Need to enable ServiceTickets fetching");
- return Tickets_->GetTicket(dst);
- }
-
- TString TTvmClient::GetServiceTicketFor(const TTvmId dst) const {
- Y_ENSURE_EX(Tickets_, TBrokenTvmClientSettings()
- << "Need to enable ServiceTickets fetching");
- return Tickets_->GetTicket(dst);
- }
-
- TCheckedServiceTicket TTvmClient::CheckServiceTicket(TStringBuf ticket) const {
- Y_ENSURE_EX(Service_, TBrokenTvmClientSettings()
- << "Need to use TClientSettings::EnableServiceTicketChecking()");
-
- TCheckedServiceTicket res = Service_->Check(ticket);
- if (SrcChecker_ && res) {
- return SrcChecker_->Check(std::move(res));
- }
- return res;
- }
-
- TCheckedUserTicket TTvmClient::CheckUserTicket(TStringBuf ticket, TMaybe<EBlackboxEnv> overrideEnv) const {
- Y_ENSURE_EX(User_, TBrokenTvmClientSettings()
- << "Need to use TClientSettings::EnableUserTicketChecking()");
-
- TCheckedUserTicket res = User_->Check(ticket, overrideEnv);
- if (DefaultUidChecker_ && res) {
- return DefaultUidChecker_->Check(std::move(res));
- }
- return User_->Check(ticket, overrideEnv);
- }
-
- NRoles::TRolesPtr TTvmClient::GetRoles() const {
- Y_ENSURE(Updater_);
- return Updater_->GetRoles();
- }
-}
diff --git a/library/cpp/tvmauth/client/facade.h b/library/cpp/tvmauth/client/facade.h
deleted file mode 100644
index 34d4b11a00..0000000000
--- a/library/cpp/tvmauth/client/facade.h
+++ /dev/null
@@ -1,119 +0,0 @@
-#pragma once
-
-#include "misc/async_updater.h"
-#include "misc/api/settings.h"
-#include "misc/tool/settings.h"
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-namespace NTvmAuth::NInternal {
- class TClientCaningKnife;
-}
-
-namespace NTvmAuth {
- class TDefaultUidChecker;
- class TServiceTicketGetter;
- class TServiceTicketChecker;
- class TSrcChecker;
- class TUserTicketChecker;
-
- /*!
- * Long lived thread-safe object for interacting with TVM.
- * In 99% cases TvmClient shoud be created at service startup and live for the whole process lifetime.
- */
- class TTvmClient {
- public:
- /*!
- * Uses local http-interface to get state: http://localhost/tvm/.
- * This interface can be provided with tvmtool (local daemon) or Qloud/YP (local http api in container).
- * See more: https://wiki.yandex-team.ru/passport/tvm2/tvm-daemon/.
- *
- * Starts thread for updating of in-memory cache in background
- * @param settings
- * @param logger is usefull for monitoring and debuging
- */
- TTvmClient(const NTvmTool::TClientSettings& settings, TLoggerPtr logger);
-
- /*!
- * Uses general way to get state: https://tvm-api.yandex.net.
- * It is not recomended for Qloud/YP.
- *
- * Starts thread for updating of in-memory cache in background
- * Reads cache from disk if specified
- * @param settings
- * @param logger is usefull for monitoring and debuging
- */
- TTvmClient(const NTvmApi::TClientSettings& settings, TLoggerPtr logger);
-
- /*!
- * Feel free to use custom updating logic in tests
- */
- TTvmClient(TAsyncUpdaterPtr updater);
-
- TTvmClient(TTvmClient&&);
- ~TTvmClient();
- TTvmClient& operator=(TTvmClient&&);
-
- /*!
- * You should trigger your monitoring if status is not Ok.
- * It will be unable to operate if status is Error.
- * Description: https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/client/README.md#high-level-interface
- * @return Current status of client.
- */
- TClientStatus GetStatus() const;
-
- /*!
- * Some tools for monitoring
- */
-
- TInstant GetUpdateTimeOfPublicKeys() const;
- TInstant GetUpdateTimeOfServiceTickets() const;
- TInstant GetInvalidationTimeOfPublicKeys() const;
- TInstant GetInvalidationTimeOfServiceTickets() const;
-
- /*!
- * Requires fetchinig options (from TClientSettings or Qloud/YP/tvmtool settings)
- * Can throw exception if cache is invalid or wrong config
- *
- * Alias is local label for TvmID
- * which can be used to avoid this number in every checking case in code.
- * @param dst
- */
- TString GetServiceTicketFor(const TClientSettings::TAlias& dst) const;
- TString GetServiceTicketFor(const TTvmId dst) const;
-
- /*!
- * For TTvmApi::TClientSettings: checking must be enabled in TClientSettings
- * Can throw exception if checking was not enabled in settings
- *
- * ServiceTicket contains src: you should check it by yourself with ACL
- * @param ticket
- */
- TCheckedServiceTicket CheckServiceTicket(TStringBuf ticket) const;
-
- /*!
- * Requires blackbox enviroment (from TClientSettings or Qloud/YP/tvmtool settings)
- * Can throw exception if checking was not enabled in settings
- * @param ticket
- * @param overrideEnv allowes you to override env from settings
- */
- TCheckedUserTicket CheckUserTicket(TStringBuf ticket, TMaybe<EBlackboxEnv> overrideEnv = {}) const;
-
- /*!
- * Under construction now. It is unusable.
- * PASSP-30283
- */
- NRoles::TRolesPtr GetRoles() const;
-
- private:
- TAsyncUpdaterPtr Updater_;
- THolder<TServiceTicketGetter> Tickets_;
- THolder<TServiceTicketChecker> Service_;
- THolder<TUserTicketChecker> User_;
- THolder<TSrcChecker> SrcChecker_;
- THolder<TDefaultUidChecker> DefaultUidChecker_;
-
- friend class NInternal::TClientCaningKnife;
- };
-}
diff --git a/library/cpp/tvmauth/client/logger.cpp b/library/cpp/tvmauth/client/logger.cpp
deleted file mode 100644
index bd63773cdf..0000000000
--- a/library/cpp/tvmauth/client/logger.cpp
+++ /dev/null
@@ -1,12 +0,0 @@
-#include "logger.h"
-
-#include <util/datetime/base.h>
-#include <util/generic/string.h>
-
-namespace NTvmAuth {
- void TCerrLogger::Log(int lvl, const TString& msg) {
- if (lvl > Level_)
- return;
- Cerr << TInstant::Now().ToStringLocal() << " lvl=" << lvl << " msg: " << msg << "\n";
- }
-}
diff --git a/library/cpp/tvmauth/client/logger.h b/library/cpp/tvmauth/client/logger.h
deleted file mode 100644
index 6f3718a2aa..0000000000
--- a/library/cpp/tvmauth/client/logger.h
+++ /dev/null
@@ -1,59 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-
-namespace NTvmAuth {
- class ILogger: public TAtomicRefCount<ILogger> {
- public:
- virtual ~ILogger() = default;
-
- void Debug(const TString& msg) {
- Log(7, msg);
- }
-
- void Info(const TString& msg) {
- Log(6, msg);
- }
-
- void Warning(const TString& msg) {
- Log(4, msg);
- }
-
- void Error(const TString& msg) {
- Log(3, msg);
- }
-
- protected:
- /*!
- * Log event
- * @param lvl is syslog level: 0(Emergency) ... 7(Debug)
- * @param msg
- */
- virtual void Log(int lvl, const TString& msg) = 0;
- };
-
- class TCerrLogger: public ILogger {
- public:
- TCerrLogger(int level)
- : Level_(level)
- {
- }
-
- void Log(int lvl, const TString& msg) override;
-
- private:
- const int Level_;
- };
-
- using TLoggerPtr = TIntrusivePtr<ILogger>;
-
- class TDevNullLogger: public ILogger {
- public:
- static TLoggerPtr IAmBrave() {
- return MakeIntrusive<TDevNullLogger>();
- }
-
- void Log(int, const TString&) override {
- }
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.cpp b/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.cpp
deleted file mode 100644
index 6ec15c0e88..0000000000
--- a/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-#include "tvm_client.h"
-
-#include <util/string/builder.h>
-
-namespace NTvmAuth::NDynamicClient {
- TAsyncUpdaterPtr TTvmClient::Create(const NTvmApi::TClientSettings& settings, TLoggerPtr logger) {
- Y_ENSURE_EX(logger, TNonRetriableException() << "Logger is required");
- THolder<TTvmClient> p(new TTvmClient(settings, std::move(logger)));
- p->Init();
- p->StartWorker();
- return p.Release();
- }
-
- NThreading::TFuture<TAddResponse> TTvmClient::Add(TDsts&& dsts) {
- if (dsts.empty()) {
- LogDebug("Adding dst: got empty task");
- return NThreading::MakeFuture<TAddResponse>(TAddResponse{});
- }
-
- const size_t size = dsts.size();
- const ui64 id = ++TaskIds_;
- NThreading::TPromise<TAddResponse> promise = NThreading::NewPromise<TAddResponse>();
-
- TaskQueue_.Enqueue(TTask{id, promise, std::move(dsts)});
-
- LogDebug(TStringBuilder() << "Adding dst: got task #" << id << " with " << size << " dsts");
- return promise.GetFuture();
- }
-
- std::optional<TString> TTvmClient::GetOptionalServiceTicketFor(const TTvmId dst) {
- TServiceTicketsPtr tickets = GetCachedServiceTickets();
- Y_ENSURE_EX(tickets,
- TBrokenTvmClientSettings()
- << "Need to enable fetching of service tickets in settings");
-
- auto it = tickets->TicketsById.find(dst);
- if (it != tickets->TicketsById.end()) {
- return it->second;
- }
-
- it = tickets->ErrorsById.find(dst);
- if (it != tickets->ErrorsById.end()) {
- ythrow TMissingServiceTicket()
- << "Failed to get ticket for '" << dst << "': "
- << it->second;
- }
-
- return {};
- }
-
- TTvmClient::TTvmClient(const NTvmApi::TClientSettings& settings, TLoggerPtr logger)
- : TBase(settings, logger)
- {
- }
-
- TTvmClient::~TTvmClient() {
- TBase::StopWorker();
- }
-
- void TTvmClient::Worker() {
- TBase::Worker();
- ProcessTasks();
- }
-
- void TTvmClient::ProcessTasks() {
- TaskQueue_.DequeueAll(&Tasks_);
- if (Tasks_.empty()) {
- return;
- }
-
- TDsts required;
- for (const TTask& task : Tasks_) {
- for (const auto& dst : task.Dsts) {
- required.insert(dst);
- }
- }
-
- TServiceTicketsPtr cache = UpdateMissingServiceTickets(required);
-
- for (TTask& task : Tasks_) {
- try {
- SetResponseForTask(task, *cache);
- } catch (const std::exception& e) {
- LogError(TStringBuilder()
- << "Adding dst: task #" << task.Id << ": exception: " << e.what());
- } catch (...) {
- LogError(TStringBuilder()
- << "Adding dst: task #" << task.Id << ": exception: " << CurrentExceptionMessage());
- }
- }
-
- Tasks_.clear();
- }
-
- static const TString UNKNOWN = "Unknown reason";
- void TTvmClient::SetResponseForTask(TTvmClient::TTask& task, const TServiceTickets& cache) {
- if (task.Promise.HasValue()) {
- LogWarning(TStringBuilder() << "Adding dst: task #" << task.Id << " already has value");
- return;
- }
-
- TAddResponse response;
-
- for (const auto& dst : task.Dsts) {
- if (cache.TicketsById.contains(dst.Id)) {
- AddDstToSettings(dst);
- response.emplace(dst, TDstResponse{EDstStatus::Success, TString()});
-
- LogDebug(TStringBuilder() << "Adding dst: task #" << task.Id
- << ": dst=" << dst.Id << " got ticket");
- continue;
- }
-
- auto it = cache.ErrorsById.find(dst.Id);
- const TString& error = it == cache.ErrorsById.end() ? UNKNOWN : it->second;
- response.emplace(dst, TDstResponse{EDstStatus::Fail, error});
-
- LogWarning(TStringBuilder() << "Adding dst: task #" << task.Id
- << ": dst=" << dst.Id
- << " failed to get ticket: " << error);
- }
-
- LogDebug(TStringBuilder() << "Adding dst: task #" << task.Id << ": set value");
- task.Promise.SetValue(std::move(response));
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.h b/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.h
deleted file mode 100644
index 58ed953b63..0000000000
--- a/library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.h
+++ /dev/null
@@ -1,60 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/misc/api/threaded_updater.h>
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/map.h>
-#include <util/thread/lfqueue.h>
-
-#include <optional>
-
-namespace NTvmAuth::NDynamicClient {
- enum class EDstStatus {
- Success,
- Fail,
- };
-
- struct TDstResponse {
- EDstStatus Status = EDstStatus::Fail;
- TString Error;
-
- bool operator==(const TDstResponse& o) const {
- return Status == o.Status && Error == o.Error;
- }
- };
-
- using TDsts = NTvmApi::TDstSet;
- using TAddResponse = TMap<NTvmApi::TClientSettings::TDst, TDstResponse>;
-
- class TTvmClient: public NTvmApi::TThreadedUpdater {
- public:
- static TAsyncUpdaterPtr Create(const NTvmApi::TClientSettings& settings, TLoggerPtr logger);
- virtual ~TTvmClient();
-
- NThreading::TFuture<TAddResponse> Add(TDsts&& dsts);
- std::optional<TString> GetOptionalServiceTicketFor(const TTvmId dst);
-
- protected: // for tests
- struct TTask {
- ui64 Id = 0;
- NThreading::TPromise<TAddResponse> Promise;
- TDsts Dsts;
- };
-
- using TBase = NTvmApi::TThreadedUpdater;
-
- protected: // for tests
- TTvmClient(const NTvmApi::TClientSettings& settings, TLoggerPtr logger);
-
- void Worker() override;
- void ProcessTasks();
-
- void SetResponseForTask(TTask& task, const TServiceTickets& cache);
-
- private:
- std::atomic<ui64> TaskIds_ = {0};
- TLockFreeQueue<TTask> TaskQueue_;
- TVector<TTask> Tasks_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/api/dynamic_dst/ut/tvm_client_ut.cpp b/library/cpp/tvmauth/client/misc/api/dynamic_dst/ut/tvm_client_ut.cpp
deleted file mode 100644
index 89403c15e4..0000000000
--- a/library/cpp/tvmauth/client/misc/api/dynamic_dst/ut/tvm_client_ut.cpp
+++ /dev/null
@@ -1,635 +0,0 @@
-#include <library/cpp/tvmauth/client/misc/api/dynamic_dst/tvm_client.h>
-
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/stream/file.h>
-#include <util/system/fs.h>
-
-#include <regex>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NDynamicClient;
-
-Y_UNIT_TEST_SUITE(DynamicClient) {
- static const std::regex TIME_REGEX(R"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d{6}Z)");
- static const TString CACHE_DIR = "./tmp/";
-
- static void WriteFile(TString name, TStringBuf body, TInstant time) {
- NFs::Remove(CACHE_DIR + name);
- TFileOutput f(CACHE_DIR + name);
- f << TDiskWriter::PrepareData(time, body);
- }
-
- static void CleanCache() {
- NFs::RemoveRecursive(CACHE_DIR);
- NFs::MakeDirectoryRecursive(CACHE_DIR);
- }
-
- class TLogger: public NTvmAuth::ILogger {
- public:
- void Log(int lvl, const TString& msg) override {
- Cout << TInstant::Now() << " lvl=" << lvl << " msg: " << msg << "\n";
- Stream << lvl << ": " << msg << Endl;
- }
-
- TStringStream Stream;
- };
-
- class TOfflineUpdater: public NDynamicClient::TTvmClient {
- public:
- TOfflineUpdater(const NTvmApi::TClientSettings& settings,
- TIntrusivePtr<TLogger> l,
- bool fail = true,
- std::vector<TString> tickets = {})
- : TTvmClient(settings, l)
- , Fail(fail)
- , Tickets(std::move(tickets))
- {
- Init();
- ExpBackoff_.SetEnabled(false);
- }
-
- NUtils::TFetchResult FetchServiceTicketsFromHttp(const TString& req) const override {
- if (Fail) {
- throw yexception() << "tickets: alarm";
- }
-
- TString response;
- if (!Tickets.empty()) {
- response = Tickets.front();
- Tickets.erase(Tickets.begin());
- }
-
- Cout << "*** FetchServiceTicketsFromHttp. request: " << req << ". response: " << response << Endl;
- return {200, {}, "/2/ticket", response, ""};
- }
-
- NUtils::TFetchResult FetchPublicKeysFromHttp() const override {
- if (Fail) {
- throw yexception() << "keysalarm";
- }
- Cout << "*** FetchPublicKeysFromHttp" << Endl;
- return {200, {}, "/2/keys", PublicKeys, ""};
- }
-
- using TTvmClient::GetDsts;
- using TTvmClient::ProcessTasks;
- using TTvmClient::SetResponseForTask;
- using TTvmClient::Worker;
-
- bool Fail = true;
- TString PublicKeys = NUnittest::TVMKNIFE_PUBLIC_KEYS;
- mutable std::vector<TString> Tickets;
- };
-
- Y_UNIT_TEST(StartWithIncompleteTicketsSet) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}}, false);
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
-
- {
- TOfflineUpdater client(s,
- l,
- false,
- {
- R"({"213" : { "error" : "some error"}})",
- R"({"123" : { "ticket" : "service_ticket_3"}})",
- });
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(213));
-
- NThreading::TFuture<TAddResponse> fut = client.Add({123});
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
-
- UNIT_ASSERT(fut.HasValue());
- TAddResponse resp{
- {123, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp, fut.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(client.GetOptionalServiceTicketFor(213), TMissingServiceTicket, "some error");
- }
- }
-
- Y_UNIT_TEST(StartWithEmptyTicketsSet) {
- CleanCache();
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"kolmo", 213}}, false);
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
-
- {
- TOfflineUpdater client(s,
- l,
- false,
- {
- R"({"213" : { "error" : "some error"}})",
- R"({"123" : { "ticket" : "3:serv:CBAQ__________9_IgYIlJEGEHs:CcafYQH-FF5XaXMuJrgLZj98bIC54cs1ZkcFS9VV_9YM9iOM_0PXCtMkdg85rFjxE_BMpg7bE8ZuoqNfdw0FPt0BAKNeISwlydj4o0IjY82--LZBpP8CRn-EpAnkRaDShdlfrcF2pk1SSmEX8xdyZVQEnkUPY0cHGlFnu231vnE"}})",
- });
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT_EXCEPTION_CONTAINS(client.GetOptionalServiceTicketFor(213), TMissingServiceTicket, "some error");
-
- NThreading::TFuture<TAddResponse> fut = client.Add({123});
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::IncompleteTicketsSet, client.GetStatus());
-
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
-
- UNIT_ASSERT(fut.HasValue());
- TAddResponse resp{
- {123, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp, fut.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(client.GetOptionalServiceTicketFor(213), TMissingServiceTicket, "some error");
- }
- };
- Y_UNIT_TEST(StartWithIncompleteCacheAndAdd) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry: ServiceTickets: tickets: alarm");
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "4: Failed to get ServiceTickets: tickets: alarm\n"
- << "4: Failed to get ServiceTickets: tickets: alarm\n"
- << "4: Failed to get ServiceTickets: tickets: alarm\n"
- << "4: Failed to update service tickets: tickets: alarm\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- l->Stream.Str().clear();
-
- {
- TOfflineUpdater client(s,
- l,
- false,
- {
- R"({"213" : { "ticket" : "service_ticket_2"}})",
- R"({"123" : { "ticket" : "service_ticket_3"}})",
- });
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
-
- NThreading::TFuture<TAddResponse> fut = client.Add({123});
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
-
- UNIT_ASSERT(fut.HasValue());
- TAddResponse resp{
- {123, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp, fut.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 1 destination(s)\n"
- << "6: Cache was partly updated with 1 service ticket(s). total: 2\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: got task #1 with 1 dsts\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 1 destination(s)\n"
- << "6: Cache was partly updated with 1 service ticket(s). total: 3\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: task #1: dst=123 got ticket\n"
- << "7: Adding dst: task #1: set value\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithCacheAndAdd) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater client(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
-
- client.Fail = false;
- client.Tickets = {
- R"({"123" : { "ticket" : "service_ticket_3"}, "213" : { "ticket" : "service_ticket_2"}})",
- };
- NThreading::TFuture<TAddResponse> fut = client.Add({123, 213});
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
-
- UNIT_ASSERT(fut.HasValue());
- TAddResponse resp{
- {123, {EDstStatus::Success, ""}},
- {213, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp, fut.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Adding dst: got task #1 with 2 dsts\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was partly updated with 2 service ticket(s). total: 3\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: task #1: dst=123 got ticket\n"
- << "7: Adding dst: task #1: dst=213 got ticket\n"
- << "7: Adding dst: task #1: set value\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithCacheAndAddSeveral) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater client(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
-
- client.Fail = false;
- client.Tickets = {
- R"({"123" : { "ticket" : "service_ticket_3"}, "213" : { "ticket" : "service_ticket_2"}})",
- };
- NThreading::TFuture<TAddResponse> fut1 = client.Add({123});
- NThreading::TFuture<TAddResponse> fut2 = client.Add({213});
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
-
- UNIT_ASSERT(fut1.HasValue());
- TAddResponse resp1{
- {123, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp1, fut1.GetValue());
-
- UNIT_ASSERT(fut2.HasValue());
- TAddResponse resp2{
- {213, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp2, fut2.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Adding dst: got task #1 with 1 dsts\n"
- << "7: Adding dst: got task #2 with 1 dsts\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was partly updated with 2 service ticket(s). total: 3\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: task #1: dst=123 got ticket\n"
- << "7: Adding dst: task #1: set value\n"
- << "7: Adding dst: task #2: dst=213 got ticket\n"
- << "7: Adding dst: task #2: set value\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithCacheAndAddSeveralWithErrors) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater client(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
-
- UNIT_ASSERT(client.GetOptionalServiceTicketFor(19));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIKhCUkQY:CX",
- *client.GetOptionalServiceTicketFor(19));
- UNIT_ASSERT(!client.GetOptionalServiceTicketFor(456));
-
- client.Fail = false;
- client.Tickets = {
- R"({
- "123" : { "ticket" : "service_ticket_3"},
- "213" : { "ticket" : "service_ticket_2"},
- "456" : { "error" : "error_3"}
- })",
- };
- NThreading::TFuture<TAddResponse> fut1 = client.Add({123, 213});
- NThreading::TFuture<TAddResponse> fut2 = client.Add({213, 456});
-
- client.Worker();
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(456));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(456));
-
- UNIT_ASSERT(client.GetOptionalServiceTicketFor(19));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIKhCUkQY:CX",
- *client.GetOptionalServiceTicketFor(19));
- UNIT_ASSERT_EXCEPTION_CONTAINS(client.GetOptionalServiceTicketFor(456),
- TMissingServiceTicket,
- "Failed to get ticket for '456': error_3");
-
- UNIT_ASSERT(fut1.HasValue());
- TAddResponse resp1{
- {123, {EDstStatus::Success, ""}},
- {213, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp1, fut1.GetValue());
-
- UNIT_ASSERT(fut2.HasValue());
- TAddResponse resp2{
- {213, {EDstStatus::Success, ""}},
- {456, {EDstStatus::Fail, "error_3"}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp2, fut2.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Adding dst: got task #1 with 2 dsts\n"
- << "7: Adding dst: got task #2 with 2 dsts\n"
- << "7: Response with service tickets for 3 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 3 destination(s)\n"
- << "3: Failed to get service ticket for dst=456: error_3\n"
- << "6: Cache was partly updated with 2 service ticket(s). total: 3\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: task #1: dst=123 got ticket\n"
- << "7: Adding dst: task #1: dst=213 got ticket\n"
- << "7: Adding dst: task #1: set value\n"
- << "7: Adding dst: task #2: dst=213 got ticket\n"
- << "4: Adding dst: task #2: dst=456 failed to get ticket: error_3\n"
- << "7: Adding dst: task #2: set value\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(WithException) {
- TInstant now = TInstant::Now();
- CleanCache();
- WriteFile("./service_tickets",
- R"({"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})"
- "\t100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater client(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
-
- client.Fail = false;
- client.Tickets = {
- R"({
- "123" : { "ticket" : "service_ticket_3"},
- "213" : { "ticket" : "service_ticket_2"},
- "456" : { "error" : "error_3"},
- "789" : { "ticket" : "service_ticket_4"}
- })",
- };
- NThreading::TFuture<TAddResponse> fut1 = client.Add({123, 213});
- NThreading::TFuture<TAddResponse> fut2 = client.Add({213, 456});
- NThreading::TFuture<TAddResponse> fut3 = client.Add({789});
-
- fut2.Subscribe([](const auto&) {
- throw yexception() << "planed exc";
- });
- fut3.Subscribe([](const auto&) {
- throw 5;
- });
-
- UNIT_ASSERT_NO_EXCEPTION(client.Worker());
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, client.GetStatus());
-
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(19));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(213));
- UNIT_ASSERT(client.GetCachedServiceTickets()->TicketsById.contains(123));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->TicketsById.contains(456));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(19));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(213));
- UNIT_ASSERT(!client.GetCachedServiceTickets()->ErrorsById.contains(123));
- UNIT_ASSERT(client.GetCachedServiceTickets()->ErrorsById.contains(456));
-
- UNIT_ASSERT(fut1.HasValue());
- TAddResponse resp1{
- {123, {EDstStatus::Success, ""}},
- {213, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp1, fut1.GetValue());
-
- UNIT_ASSERT(fut2.HasValue());
- TAddResponse resp2{
- {213, {EDstStatus::Success, ""}},
- {456, {EDstStatus::Fail, "error_3"}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp2, fut2.GetValue());
-
- UNIT_ASSERT(fut3.HasValue());
- TAddResponse resp3{
- {789, {EDstStatus::Success, ""}},
- };
- UNIT_ASSERT_VALUES_EQUAL(resp3, fut3.GetValue());
-
- UNIT_ASSERT(client.Tickets.empty());
-
- TDsts dsts{19, 123, 213, 789};
- UNIT_ASSERT_VALUES_EQUAL(dsts, client.GetDsts());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Adding dst: got task #1 with 2 dsts\n"
- << "7: Adding dst: got task #2 with 2 dsts\n"
- << "7: Adding dst: got task #3 with 1 dsts\n"
- << "7: Response with service tickets for 4 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 4 destination(s)\n"
- << "3: Failed to get service ticket for dst=456: error_3\n"
- << "6: Cache was partly updated with 3 service ticket(s). total: 4\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Adding dst: task #1: dst=123 got ticket\n"
- << "7: Adding dst: task #1: dst=213 got ticket\n"
- << "7: Adding dst: task #1: set value\n"
- << "7: Adding dst: task #2: dst=213 got ticket\n"
- << "4: Adding dst: task #2: dst=456 failed to get ticket: error_3\n"
- << "7: Adding dst: task #2: set value\n"
- << "3: Adding dst: task #2: exception: planed exc\n"
- << "7: Adding dst: task #3: dst=789 got ticket\n"
- << "7: Adding dst: task #3: set value\n"
- << "3: Adding dst: task #3: exception: unknown error\n",
- l->Stream.Str());
- }
-}
-
-template <>
-void Out<NTvmAuth::NDynamicClient::TDstResponse>(IOutputStream& out, const NTvmAuth::NDynamicClient::TDstResponse& m) {
- out << m.Status << " (" << m.Error << ")";
-}
-
-template <>
-void Out<NTvmAuth::NTvmApi::TClientSettings::TDst>(IOutputStream& out, const NTvmAuth::NTvmApi::TClientSettings::TDst& m) {
- out << m.Id;
-}
diff --git a/library/cpp/tvmauth/client/misc/api/retry_settings.h b/library/cpp/tvmauth/client/misc/api/retry_settings.h
deleted file mode 100644
index 607b230811..0000000000
--- a/library/cpp/tvmauth/client/misc/api/retry_settings.h
+++ /dev/null
@@ -1,33 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/misc/exponential_backoff.h>
-
-namespace NTvmAuth::NTvmApi {
- struct TRetrySettings {
- TExponentialBackoff::TSettings BackoffSettings = {
- TDuration::Seconds(0),
- TDuration::Minutes(1),
- 2,
- 0.5,
- };
- TDuration MaxRandomSleepDefault = TDuration::Seconds(5);
- TDuration MaxRandomSleepWhenOk = TDuration::Minutes(1);
- ui32 RetriesOnStart = 3;
- ui32 RetriesInBackground = 2;
- TDuration WorkerAwakingPeriod = TDuration::Seconds(10);
- ui32 DstsLimit = 300;
- TDuration RolesUpdatePeriod = TDuration::Minutes(10);
- TDuration RolesWarnPeriod = TDuration::Minutes(20);
-
- bool operator==(const TRetrySettings& o) const {
- return BackoffSettings == o.BackoffSettings &&
- MaxRandomSleepDefault == o.MaxRandomSleepDefault &&
- MaxRandomSleepWhenOk == o.MaxRandomSleepWhenOk &&
- RetriesOnStart == o.RetriesOnStart &&
- WorkerAwakingPeriod == o.WorkerAwakingPeriod &&
- DstsLimit == o.DstsLimit &&
- RolesUpdatePeriod == o.RolesUpdatePeriod &&
- RolesWarnPeriod == o.RolesWarnPeriod;
- }
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/api/roles_fetcher.cpp b/library/cpp/tvmauth/client/misc/api/roles_fetcher.cpp
deleted file mode 100644
index 8f4b359e8c..0000000000
--- a/library/cpp/tvmauth/client/misc/api/roles_fetcher.cpp
+++ /dev/null
@@ -1,164 +0,0 @@
-#include "roles_fetcher.h"
-
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-#include <library/cpp/tvmauth/client/misc/roles/decoder.h>
-#include <library/cpp/tvmauth/client/misc/roles/parser.h>
-
-#include <library/cpp/http/misc/httpcodes.h>
-#include <library/cpp/string_utils/quote/quote.h>
-
-#include <util/string/builder.h>
-#include <util/string/join.h>
-
-namespace NTvmAuth::NTvmApi {
- static TString CreatePath(const TString& dir, const TString& file) {
- return dir.EndsWith("/")
- ? dir + file
- : dir + "/" + file;
- }
-
- TRolesFetcher::TRolesFetcher(const TRolesFetcherSettings& settings, TLoggerPtr logger)
- : Settings_(settings)
- , Logger_(logger)
- , CacheFilePath_(CreatePath(Settings_.CacheDir, "roles"))
- {
- Client_ = std::make_unique<TKeepAliveHttpClient>(
- Settings_.TiroleHost,
- Settings_.TirolePort,
- Settings_.Timeout,
- Settings_.Timeout);
- }
-
- TInstant TRolesFetcher::ReadFromDisk() {
- TDiskReader dr(CacheFilePath_, Logger_.Get());
- if (!dr.Read()) {
- return {};
- }
-
- std::pair<TString, TString> data = ParseDiskFormat(dr.Data());
- if (data.second != Settings_.IdmSystemSlug) {
- Logger_->Warning(
- TStringBuilder() << "Roles in disk cache are for another slug (" << data.second
- << "). Self=" << Settings_.IdmSystemSlug);
- return {};
- }
-
- CurrentRoles_.Set(NRoles::TParser::Parse(std::make_shared<TString>(std::move(data.first))));
- Logger_->Debug(
- TStringBuilder() << "Succeed to read roles with revision "
- << CurrentRoles_.Get()->GetMeta().Revision
- << " from " << CacheFilePath_);
-
- return dr.Time();
- }
-
- bool TRolesFetcher::AreRolesOk() const {
- return bool(GetCurrentRoles());
- }
-
- bool TRolesFetcher::IsTimeToUpdate(const TRetrySettings& settings, TDuration sinceUpdate) {
- return settings.RolesUpdatePeriod < sinceUpdate;
- }
-
- bool TRolesFetcher::ShouldWarn(const TRetrySettings& settings, TDuration sinceUpdate) {
- return settings.RolesWarnPeriod < sinceUpdate;
- }
-
- NUtils::TFetchResult TRolesFetcher::FetchActualRoles(const TString& serviceTicket) {
- TStringStream out;
- THttpHeaders outHeaders;
-
- TRequest req = CreateTiroleRequest(serviceTicket);
- TKeepAliveHttpClient::THttpCode code = Client_->DoGet(
- req.Url,
- &out,
- req.Headers,
- &outHeaders);
-
- const THttpInputHeader* reqId = outHeaders.FindHeader("X-Request-Id");
-
- Logger_->Debug(
- TStringBuilder() << "Succeed to perform request for roles to " << Settings_.TiroleHost
- << " (request_id=" << (reqId ? reqId->Value() : "")
- << "). code=" << code);
-
- return {code, std::move(outHeaders), "/v1/get_actual_roles", out.Str(), {}};
- }
-
- void TRolesFetcher::Update(NUtils::TFetchResult&& fetchResult, TInstant now) {
- if (fetchResult.Code == HTTP_NOT_MODIFIED) {
- Y_ENSURE(CurrentRoles_.Get(),
- "tirole did not return any roles because current roles are actual,"
- " but there are no roles in memory - this should never happen");
- return;
- }
-
- Y_ENSURE(fetchResult.Code == HTTP_OK,
- "Unexpected code from tirole: " << fetchResult.Code << ". " << fetchResult.Response);
-
- const THttpInputHeader* codec = fetchResult.Headers.FindHeader("X-Tirole-Compression");
- const TStringBuf codecBuf = codec ? codec->Value() : "";
-
- NRoles::TRawPtr blob;
- try {
- blob = std::make_shared<TString>(NRoles::TDecoder::Decode(
- codecBuf,
- std::move(fetchResult.Response)));
- } catch (const std::exception& e) {
- throw yexception() << "Failed to decode blob with codec '" << codecBuf
- << "': " << e.what();
- }
-
- CurrentRoles_.Set(NRoles::TParser::Parse(blob));
-
- Logger_->Debug(
- TStringBuilder() << "Succeed to update roles with revision "
- << CurrentRoles_.Get()->GetMeta().Revision);
-
- TDiskWriter dw(CacheFilePath_, Logger_.Get());
- dw.Write(PrepareDiskFormat(*blob, Settings_.IdmSystemSlug), now);
- }
-
- NTvmAuth::NRoles::TRolesPtr TRolesFetcher::GetCurrentRoles() const {
- return CurrentRoles_.Get();
- }
-
- void TRolesFetcher::ResetConnection() {
- Client_->ResetConnection();
- }
-
- static const char DELIMETER = '\t';
-
- std::pair<TString, TString> TRolesFetcher::ParseDiskFormat(TStringBuf filebody) {
- TStringBuf slug = filebody.RNextTok(DELIMETER);
- return {TString(filebody), CGIUnescapeRet(slug)};
- }
-
- TString TRolesFetcher::PrepareDiskFormat(TStringBuf roles, TStringBuf slug) {
- TStringStream res;
- res.Reserve(roles.size() + 1 + slug.size());
- res << roles << DELIMETER << CGIEscapeRet(slug);
- return res.Str();
- }
-
- TRolesFetcher::TRequest TRolesFetcher::CreateTiroleRequest(const TString& serviceTicket) const {
- TRolesFetcher::TRequest res;
-
- TStringStream url;
- url.Reserve(512);
- url << "/v1/get_actual_roles?";
- url << "system_slug=" << CGIEscapeRet(Settings_.IdmSystemSlug) << "&";
- Settings_.ProcInfo.AddToRequest(url);
- res.Url = std::move(url.Str());
-
- res.Headers.reserve(2);
- res.Headers.emplace(XYaServiceTicket_, serviceTicket);
-
- NRoles::TRolesPtr roles = CurrentRoles_.Get();
- if (roles) {
- res.Headers.emplace(IfNoneMatch_, Join("", "\"", roles->GetMeta().Revision, "\""));
- }
-
- return res;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/api/roles_fetcher.h b/library/cpp/tvmauth/client/misc/api/roles_fetcher.h
deleted file mode 100644
index 63691223b5..0000000000
--- a/library/cpp/tvmauth/client/misc/api/roles_fetcher.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#pragma once
-
-#include "retry_settings.h"
-
-#include <library/cpp/tvmauth/client/misc/fetch_result.h>
-#include <library/cpp/tvmauth/client/misc/proc_info.h>
-#include <library/cpp/tvmauth/client/misc/utils.h>
-#include <library/cpp/tvmauth/client/misc/roles/roles.h>
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/http/simple/http_client.h>
-
-namespace NTvmAuth::NTvmApi {
- struct TRolesFetcherSettings {
- TString TiroleHost;
- ui16 TirolePort = 0;
- TString CacheDir;
- NUtils::TProcInfo ProcInfo;
- TTvmId SelfTvmId = 0;
- TString IdmSystemSlug;
- TDuration Timeout = TDuration::Seconds(30);
- };
-
- class TRolesFetcher {
- public:
- TRolesFetcher(const TRolesFetcherSettings& settings, TLoggerPtr logger);
-
- TInstant ReadFromDisk();
-
- bool AreRolesOk() const;
- static bool IsTimeToUpdate(const TRetrySettings& settings, TDuration sinceUpdate);
- static bool ShouldWarn(const TRetrySettings& settings, TDuration sinceUpdate);
-
- NUtils::TFetchResult FetchActualRoles(const TString& serviceTicket);
- void Update(NUtils::TFetchResult&& fetchResult, TInstant now = TInstant::Now());
-
- NTvmAuth::NRoles::TRolesPtr GetCurrentRoles() const;
-
- void ResetConnection();
-
- public:
- static std::pair<TString, TString> ParseDiskFormat(TStringBuf filebody);
- static TString PrepareDiskFormat(TStringBuf roles, TStringBuf slug);
-
- struct TRequest {
- TString Url;
- TKeepAliveHttpClient::THeaders Headers;
- };
- TRequest CreateTiroleRequest(const TString& serviceTicket) const;
-
- private:
- const TRolesFetcherSettings Settings_;
- const TLoggerPtr Logger_;
- const TString CacheFilePath_;
- const TString XYaServiceTicket_ = "X-Ya-Service-Ticket";
- const TString IfNoneMatch_ = "If-None-Match";
-
- NUtils::TProtectedValue<NTvmAuth::NRoles::TRolesPtr> CurrentRoles_;
-
- std::unique_ptr<TKeepAliveHttpClient> Client_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/api/settings.cpp b/library/cpp/tvmauth/client/misc/api/settings.cpp
deleted file mode 100644
index 71aad75998..0000000000
--- a/library/cpp/tvmauth/client/misc/api/settings.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-#include "settings.h"
-
-#include <util/datetime/base.h>
-#include <util/stream/file.h>
-#include <util/system/fs.h>
-
-#include <set>
-
-namespace NTvmAuth::NTvmApi {
- void TClientSettings::CheckPermissions(const TString& dir) {
- const TString name = dir + "/check.tmp";
-
- try {
- NFs::EnsureExists(dir);
-
- TFile file(name, CreateAlways | RdWr);
-
- NFs::Remove(name);
- } catch (const std::exception& e) {
- NFs::Remove(name);
- ythrow TPermissionDenied() << "Permission denied to disk cache directory: " << e.what();
- }
- }
-
- void TClientSettings::CheckValid() const {
- if (DiskCacheDir) {
- CheckPermissions(DiskCacheDir);
- }
-
- if (TStringBuf(Secret)) {
- Y_ENSURE_EX(NeedServiceTicketsFetching(),
- TBrokenTvmClientSettings() << "Secret is present but destinations list is empty. It makes no sense");
- }
- if (NeedServiceTicketsFetching()) {
- Y_ENSURE_EX(SelfTvmId != 0,
- TBrokenTvmClientSettings() << "SelfTvmId cannot be 0 if fetching of Service Tickets required");
- Y_ENSURE_EX((TStringBuf)Secret,
- TBrokenTvmClientSettings() << "Secret is required for fetching of Service Tickets");
- }
-
- if (CheckServiceTickets) {
- Y_ENSURE_EX(SelfTvmId != 0,
- TBrokenTvmClientSettings() << "SelfTvmId cannot be 0 if checking of Service Tickets required");
- }
-
- if (FetchRolesForIdmSystemSlug) {
- Y_ENSURE_EX(DiskCacheDir,
- TBrokenTvmClientSettings() << "Disk cache must be enabled to use roles: "
- "they can be heavy");
- }
-
- bool needSmth = NeedServiceTicketsFetching() ||
- IsServiceTicketCheckingRequired() ||
- IsUserTicketCheckingRequired();
- Y_ENSURE_EX(needSmth, TBrokenTvmClientSettings() << "Invalid settings: nothing to do");
-
- // Useless now: keep it here to avoid forgetting check from TDst. TODO: PASSP-35377
- for (const auto& dst : FetchServiceTicketsForDsts) {
- Y_ENSURE_EX(dst.Id != 0, TBrokenTvmClientSettings() << "TvmId cannot be 0");
- }
- // TODO: check only FetchServiceTicketsForDsts_
- // Python binding checks settings before normalization
- for (const auto& [alias, dst] : FetchServiceTicketsForDstsWithAliases) {
- Y_ENSURE_EX(dst.Id != 0, TBrokenTvmClientSettings() << "TvmId cannot be 0");
- }
- Y_ENSURE_EX(TiroleTvmId != 0, TBrokenTvmClientSettings() << "TiroleTvmId cannot be 0");
- }
-
- TClientSettings TClientSettings::CloneNormalized() const {
- TClientSettings res = *this;
-
- std::set<TTvmId> allDsts;
- for (const auto& tvmid : res.FetchServiceTicketsForDsts) {
- allDsts.insert(tvmid.Id);
- }
- for (const auto& [alias, tvmid] : res.FetchServiceTicketsForDstsWithAliases) {
- allDsts.insert(tvmid.Id);
- }
- if (FetchRolesForIdmSystemSlug) {
- allDsts.insert(res.TiroleTvmId);
- }
-
- res.FetchServiceTicketsForDsts = {allDsts.begin(), allDsts.end()};
-
- res.CheckValid();
-
- return res;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/api/settings.h b/library/cpp/tvmauth/client/misc/api/settings.h
deleted file mode 100644
index 715ab3e02c..0000000000
--- a/library/cpp/tvmauth/client/misc/api/settings.h
+++ /dev/null
@@ -1,302 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/misc/settings.h>
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-#include <library/cpp/tvmauth/type.h>
-
-#include <library/cpp/string_utils/secret_string/secret_string.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/hash.h>
-#include <util/generic/maybe.h>
-
-namespace NTvmAuth::NTvmApi {
- /**
- * Settings for TVM client. Uses https://tvm-api.yandex.net to get state.
- * At least one of them is required:
- * FetchServiceTicketsForDsts_/FetchServiceTicketsForDstsWithAliases_
- * CheckServiceTickets_
- * CheckUserTicketsWithBbEnv_
- */
- class TClientSettings: public NTvmAuth::TClientSettings {
- public:
- class TDst;
-
- /**
- * Alias is an internal name for destinations within your code.
- * You can associate a name with an tvm_id once in your code and use the name as an alias for
- * tvm_id to each calling point. Useful for several environments: prod/test/etc.
- * @example:
- * // init
- * static const TString MY_BACKEND = "my backend";
- * TDstMap map = {{MY_BACKEND, TDst(config.get("my_back_tvm_id"))}};
- * ...
- * // per request
- * TString t = tvmClient.GetServiceTicket(MY_BACKEND);
- */
- using TDstMap = THashMap<TAlias, TDst>;
- using TDstVector = TVector<TDst>;
-
- public:
- /*!
- * NOTE: Please use this option: it provides the best reliability
- * NOTE: Client requires read/write permissions
- * WARNING: The same directory can be used only:
- * - for TVM clients with the same settings
- * OR
- * - for new client replacing previous - with another config.
- * System user must be the same for processes with these clients inside.
- * Implementation doesn't provide other scenarios.
- */
- TString DiskCacheDir;
-
- // Required for Service Ticket fetching or checking
- TTvmId SelfTvmId = 0;
-
- // Options for Service Tickets fetching
- NSecretString::TSecretString Secret;
- /*!
- * Client will process both attrs:
- * FetchServiceTicketsForDsts_, FetchServiceTicketsForDstsWithAliases_
- * WARNING: It is not way to provide authorization for incoming ServiceTickets!
- * It is way only to send your ServiceTickets to your backend!
- */
- TDstVector FetchServiceTicketsForDsts;
- TDstMap FetchServiceTicketsForDstsWithAliases;
- bool IsIncompleteTicketsSetAnError = true;
-
- // Options for Service Tickets checking
- bool CheckServiceTickets = false;
-
- // Options for User Tickets checking
- TMaybe<EBlackboxEnv> CheckUserTicketsWithBbEnv;
-
- // Options for roles fetching
- TString FetchRolesForIdmSystemSlug;
- /*!
- * By default client checks src from ServiceTicket or default uid from UserTicket -
- * to prevent you from forgetting to check it yourself.
- * It does binary checks only:
- * ticket gets status NoRoles, if there is no role for src or default uid.
- * You need to check roles on your own if you have a non-binary role system or
- * you have disabled ShouldCheckSrc/ShouldCheckDefaultUid
- *
- * You may need to disable this check in the following cases:
- * - You use GetRoles() to provide verbose message (with revision).
- * Double check may be inconsistent:
- * binary check inside client uses revision of roles X - i.e. src 100500 has no role,
- * exact check in your code uses revision of roles Y - i.e. src 100500 has some roles.
- */
- bool ShouldCheckSrc = true;
- bool ShouldCheckDefaultUid = true;
-
- // Options for tests
- TString TvmHost = "https://tvm-api.yandex.net";
- ui16 TvmPort = 443;
- TString TiroleHost = "https://tirole-api.yandex.net";
- TDuration TvmSocketTimeout = TDuration::Seconds(5);
- TDuration TvmConnectTimeout = TDuration::Seconds(30);
- ui16 TirolePort = 443;
- TTvmId TiroleTvmId = TIROLE_TVMID;
-
- // for debug purposes
- TString LibVersionPrefix;
-
- void CheckValid() const;
- TClientSettings CloneNormalized() const;
-
- static inline const TTvmId TIROLE_TVMID = 2028120;
- static inline const TTvmId TIROLE_TVMID_TEST = 2026536;
-
- // DEPRECATED API
- // TODO: get rid of it: PASSP-35377
- public:
- // Deprecated: set attributes directly
- void SetSelfTvmId(TTvmId selfTvmId) {
- SelfTvmId = selfTvmId;
- }
-
- // Deprecated: set attributes directly
- void EnableServiceTicketChecking() {
- CheckServiceTickets = true;
- }
-
- // Deprecated: set attributes directly
- void EnableUserTicketChecking(EBlackboxEnv env) {
- CheckUserTicketsWithBbEnv = env;
- }
-
- // Deprecated: set attributes directly
- void SetTvmHostPort(const TString& host, ui16 port) {
- TvmHost = host;
- TvmPort = port;
- }
-
- // Deprecated: set attributes directly
- void SetTiroleHostPort(const TString& host, ui16 port) {
- TiroleHost = host;
- TirolePort = port;
- }
-
- // Deprecated: set attributes directly
- void EnableRolesFetching(const TString& systemSlug, TTvmId tiroleTvmId = TIROLE_TVMID) {
- TiroleTvmId = tiroleTvmId;
- FetchRolesForIdmSystemSlug = systemSlug;
- }
-
- // Deprecated: set attributes directly
- void DoNotCheckSrcByDefault() {
- ShouldCheckSrc = false;
- }
-
- // Deprecated: set attributes directly
- void DoNotCheckDefaultUidByDefault() {
- ShouldCheckDefaultUid = false;
- }
-
- // Deprecated: set attributes directly
- void SetDiskCacheDir(const TString& dir) {
- DiskCacheDir = dir;
- }
-
- // Deprecated: set attributes directly
- void EnableServiceTicketsFetchOptions(const TStringBuf selfSecret,
- TDstMap&& dsts,
- const bool considerIncompleteTicketsSetAsError = true) {
- IsIncompleteTicketsSetAnError = considerIncompleteTicketsSetAsError;
- Secret = selfSecret;
-
- FetchServiceTicketsForDsts = TDstVector{};
- FetchServiceTicketsForDsts.reserve(dsts.size());
- for (const auto& pair : dsts) {
- FetchServiceTicketsForDsts.push_back(pair.second);
- }
-
- FetchServiceTicketsForDstsWithAliases = std::move(dsts);
- }
-
- // Deprecated: set attributes directly
- void EnableServiceTicketsFetchOptions(const TStringBuf selfSecret,
- TDstVector&& dsts,
- const bool considerIncompleteTicketsSetAsError = true) {
- IsIncompleteTicketsSetAnError = considerIncompleteTicketsSetAsError;
- Secret = selfSecret;
- FetchServiceTicketsForDsts = std::move(dsts);
- }
-
- public:
- bool IsServiceTicketFetchingRequired() const {
- return bool(Secret.Value());
- }
-
- const TStringBuf GetSelfSecret() const {
- return Secret;
- }
-
- bool HasDstAliases() const {
- return !FetchServiceTicketsForDstsWithAliases.empty();
- }
-
- const TDstMap& GetDstAliases() const {
- return FetchServiceTicketsForDstsWithAliases;
- }
-
- const TDstVector& GetDestinations() const {
- return FetchServiceTicketsForDsts;
- }
-
- bool IsUserTicketCheckingRequired() const {
- return bool(CheckUserTicketsWithBbEnv);
- }
-
- EBlackboxEnv GetEnvForUserTickets() const {
- return *CheckUserTicketsWithBbEnv;
- }
-
- bool IsServiceTicketCheckingRequired() const {
- return CheckServiceTickets;
- }
-
- bool IsDiskCacheUsed() const {
- return bool(DiskCacheDir);
- }
-
- TString GetDiskCacheDir() const {
- return DiskCacheDir;
- }
-
- TTvmId GetSelfTvmId() const {
- return SelfTvmId;
- }
-
- const TString& GetLibVersionPrefix() const {
- return LibVersionPrefix;
- }
-
- const TString& GetTvmHost() const {
- return TvmHost;
- }
-
- ui16 GetTvmPort() const {
- return TvmPort;
- }
-
- bool IsRolesFetchingEnabled() const {
- return bool(FetchRolesForIdmSystemSlug);
- }
-
- TTvmId GetTiroleTvmId() const {
- return TiroleTvmId;
- }
-
- const TString& GetIdmSystemSlug() const {
- return FetchRolesForIdmSystemSlug;
- }
-
- const TString& GetTiroleHost() const {
- return TiroleHost;
- }
-
- ui16 GetTirolePort() const {
- return TirolePort;
- }
-
- bool NeedServiceTicketsFetching() const {
- return !FetchServiceTicketsForDsts.empty() ||
- !FetchServiceTicketsForDstsWithAliases.empty() ||
- FetchRolesForIdmSystemSlug;
- }
-
- // TODO: get rid of TDst: PASSP-35377
- class TDst {
- public:
- TDst(TTvmId id)
- : Id(id)
- {
- Y_ENSURE_EX(id != 0, TBrokenTvmClientSettings() << "TvmId cannot be 0");
- }
-
- TTvmId Id;
-
- bool operator==(const TDst& o) const {
- return Id == o.Id;
- }
-
- bool operator<(const TDst& o) const {
- return Id < o.Id;
- }
-
- public: // for python binding
- TDst()
- : Id(0)
- {
- }
- };
-
- public:
- static void CheckPermissions(const TString& dir);
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/api/threaded_updater.cpp b/library/cpp/tvmauth/client/misc/api/threaded_updater.cpp
deleted file mode 100644
index a7df49c05d..0000000000
--- a/library/cpp/tvmauth/client/misc/api/threaded_updater.cpp
+++ /dev/null
@@ -1,954 +0,0 @@
-#include "threaded_updater.h"
-
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-#include <library/cpp/tvmauth/client/misc/utils.h>
-#include <library/cpp/tvmauth/client/misc/retry_settings/v1/settings.pb.h>
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/json/json_reader.h>
-
-#include <util/stream/str.h>
-#include <util/string/builder.h>
-#include <util/string/cast.h>
-#include <util/system/thread.h>
-
-namespace NTvmAuth::NTvmApi {
- static TString CreatePublicKeysUrl(const TClientSettings& settings,
- const NUtils::TProcInfo& procInfo) {
- TStringStream s;
- s << "/2/keys";
- s << "?";
- procInfo.AddToRequest(s);
-
- s << "&get_retry_settings=yes";
-
- if (settings.GetSelfTvmId() != 0) {
- s << "&src=" << settings.GetSelfTvmId();
- }
-
- if (settings.IsUserTicketCheckingRequired()) {
- s << "&env=" << static_cast<int>(settings.GetEnvForUserTickets());
- }
-
- return s.Str();
- }
-
- TAsyncUpdaterPtr TThreadedUpdater::Create(const TClientSettings& settings, TLoggerPtr logger) {
- Y_ENSURE_EX(logger, TNonRetriableException() << "Logger is required");
- THolder<TThreadedUpdater> p(new TThreadedUpdater(settings, std::move(logger)));
- p->Init();
- p->StartWorker();
- return p.Release();
- }
-
- TThreadedUpdater::~TThreadedUpdater() {
- ExpBackoff_.SetEnabled(false);
- ExpBackoff_.Interrupt();
- StopWorker(); // Required here to avoid using of deleted members
- }
-
- TClientStatus TThreadedUpdater::GetStatus() const {
- const TClientStatus::ECode state = GetState();
- return TClientStatus(state, GetLastError(state == TClientStatus::Ok || state == TClientStatus::IncompleteTicketsSet));
- }
-
- NRoles::TRolesPtr TThreadedUpdater::GetRoles() const {
- Y_ENSURE_EX(RolesFetcher_,
- TBrokenTvmClientSettings() << "Roles were not configured in settings");
- return RolesFetcher_->GetCurrentRoles();
- }
-
- TClientStatus::ECode TThreadedUpdater::GetState() const {
- const TInstant now = TInstant::Now();
-
- if (Settings_.IsServiceTicketFetchingRequired()) {
- if (AreServiceTicketsInvalid(now)) {
- return TClientStatus::Error;
- }
- auto tickets = GetCachedServiceTickets();
- if (!tickets) {
- return TClientStatus::Error;
- }
- if (tickets->TicketsById.size() < Destinations_.size()) {
- if (Settings_.IsIncompleteTicketsSetAnError) {
- return TClientStatus::Error;
- } else {
- return TClientStatus::IncompleteTicketsSet;
- }
- }
- }
- if ((Settings_.IsServiceTicketCheckingRequired() || Settings_.IsUserTicketCheckingRequired()) && ArePublicKeysInvalid(now)) {
- return TClientStatus::Error;
- }
-
- const TDuration sincePublicKeysUpdate = now - GetUpdateTimeOfPublicKeys();
- const TDuration sinceServiceTicketsUpdate = now - GetUpdateTimeOfServiceTickets();
- const TDuration sinceRolesUpdate = now - GetUpdateTimeOfRoles();
-
- if (Settings_.IsServiceTicketFetchingRequired() && sinceServiceTicketsUpdate > ServiceTicketsDurations_.Expiring) {
- return TClientStatus::Warning;
- }
- if ((Settings_.IsServiceTicketCheckingRequired() || Settings_.IsUserTicketCheckingRequired()) &&
- sincePublicKeysUpdate > PublicKeysDurations_.Expiring)
- {
- return TClientStatus::Warning;
- }
- if (RolesFetcher_ && TRolesFetcher::ShouldWarn(RetrySettings_, sinceRolesUpdate)) {
- return TClientStatus::Warning;
- }
-
- return TClientStatus::Ok;
- }
-
- TThreadedUpdater::TThreadedUpdater(const TClientSettings& settings, TLoggerPtr logger)
- : TThreadedUpdaterBase(
- TRetrySettings{}.WorkerAwakingPeriod,
- std::move(logger),
- settings.GetTvmHost(),
- settings.GetTvmPort(),
- settings.TvmSocketTimeout,
- settings.TvmConnectTimeout)
- , ExpBackoff_(RetrySettings_.BackoffSettings)
- , Settings_(settings.CloneNormalized())
- , ProcInfo_(NUtils::TProcInfo::Create(Settings_.GetLibVersionPrefix()))
- , PublicKeysUrl_(CreatePublicKeysUrl(Settings_, ProcInfo_))
- , DstAliases_(MakeAliasMap(Settings_))
- , Headers_({{"Content-Type", "application/x-www-form-urlencoded"}})
- , Random_(TInstant::Now().MicroSeconds())
- {
- if (Settings_.IsServiceTicketFetchingRequired()) {
- SigningContext_ = TServiceContext::SigningFactory(Settings_.GetSelfSecret());
- }
-
- if (Settings_.IsServiceTicketFetchingRequired()) {
- Destinations_ = {Settings_.GetDestinations().begin(), Settings_.GetDestinations().end()};
- }
-
- PublicKeysDurations_.RefreshPeriod = TDuration::Days(1);
- ServiceTicketsDurations_.RefreshPeriod = TDuration::Hours(1);
-
- if (Settings_.IsUserTicketCheckingRequired()) {
- SetBbEnv(Settings_.GetEnvForUserTickets());
- }
-
- if (Settings_.IsRolesFetchingEnabled()) {
- RolesFetcher_ = std::make_unique<TRolesFetcher>(
- TRolesFetcherSettings{
- Settings_.GetTiroleHost(),
- Settings_.GetTirolePort(),
- Settings_.GetDiskCacheDir(),
- ProcInfo_,
- Settings_.GetSelfTvmId(),
- Settings_.GetIdmSystemSlug(),
- },
- Logger_);
- }
-
- if (Settings_.IsDiskCacheUsed()) {
- TString path = Settings_.GetDiskCacheDir();
- if (path.back() != '/') {
- path.push_back('/');
- }
-
- if (Settings_.IsServiceTicketFetchingRequired()) {
- ServiceTicketsFilepath_ = path;
- ServiceTicketsFilepath_.append("service_tickets");
- }
-
- if (Settings_.IsServiceTicketCheckingRequired() || Settings_.IsUserTicketCheckingRequired()) {
- PublicKeysFilepath_ = path;
- PublicKeysFilepath_.append("public_keys");
- }
-
- RetrySettingsFilepath_ = path + "retry_settings";
- } else {
- LogInfo("Disk cache disabled. Please set disk cache directory in settings for best reliability");
- }
- }
-
- void TThreadedUpdater::Init() {
- ReadStateFromDisk();
- ClearErrors();
- ExpBackoff_.SetEnabled(false);
-
- // First of all try to get tickets: there are a lot of reasons to fail this request.
- // As far as disk cache usually disabled, client will not fetch keys before fail on every ctor call.
- UpdateServiceTickets();
- if (!AreServicesTicketsOk()) {
- ThrowLastError();
- }
-
- UpdatePublicKeys();
- if (!IsServiceContextOk() || !IsUserContextOk()) {
- ThrowLastError();
- }
-
- UpdateRoles();
- if (RolesFetcher_ && !RolesFetcher_->AreRolesOk()) {
- ThrowLastError();
- }
-
- Inited_ = true;
- ExpBackoff_.SetEnabled(true);
- }
-
- void TThreadedUpdater::UpdateServiceTickets() {
- if (!Settings_.IsServiceTicketFetchingRequired()) {
- return;
- }
-
- TInstant stut = GetUpdateTimeOfServiceTickets();
- try {
- if (IsTimeToUpdateServiceTickets(stut)) {
- UpdateAllServiceTickets();
- NeedFetchMissingServiceTickets_ = false;
- } else if (NeedFetchMissingServiceTickets_ && GetCachedServiceTickets()->TicketsById.size() < Destinations_.size()) {
- UpdateMissingServiceTickets(Destinations_);
- NeedFetchMissingServiceTickets_ = false;
- }
- if (AreServicesTicketsOk()) {
- ClearError(EScope::ServiceTickets);
- }
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::ServiceTickets, e.what());
- LogWarning(TStringBuilder() << "Failed to update service tickets: " << e.what());
- if (TInstant::Now() - stut > ServiceTicketsDurations_.Expiring) {
- LogError("Service tickets have not been refreshed for too long period");
- }
- }
- }
-
- void TThreadedUpdater::UpdateAllServiceTickets() {
- THttpResult st = GetServiceTicketsFromHttp(Destinations_, RetrySettings_.DstsLimit);
-
- auto oldCache = GetCachedServiceTickets();
- if (oldCache) {
- for (const auto& pair : oldCache->ErrorsById) {
- st.TicketsWithErrors.Errors.insert(pair);
- }
- }
-
- UpdateServiceTicketsCache(std::move(st.TicketsWithErrors), TInstant::Now());
- if (ServiceTicketsFilepath_) {
- DiskCacheServiceTickets_ = CreateJsonArray(st.Responses);
- TDiskWriter w(ServiceTicketsFilepath_, Logger_.Get());
- w.Write(PrepareTicketsForDisk(DiskCacheServiceTickets_, Settings_.GetSelfTvmId()));
- }
- }
-
- TServiceTicketsPtr TThreadedUpdater::UpdateMissingServiceTickets(const TDstSet& required) {
- TServiceTicketsPtr cache = GetCachedServiceTickets();
- TClientSettings::TDstVector dsts = FindMissingDsts(cache, required);
-
- if (dsts.empty()) {
- return cache;
- }
-
- THttpResult st = GetServiceTicketsFromHttp(dsts, RetrySettings_.DstsLimit);
-
- size_t gotTickets = st.TicketsWithErrors.Tickets.size();
-
- for (const auto& pair : cache->TicketsById) {
- st.TicketsWithErrors.Tickets.insert(pair);
- }
- for (const auto& pair : cache->ErrorsById) {
- st.TicketsWithErrors.Errors.insert(pair);
- }
- for (const auto& pair : st.TicketsWithErrors.Tickets) {
- st.TicketsWithErrors.Errors.erase(pair.first);
- }
-
- TServiceTicketsPtr c = UpdateServiceTicketsCachePartly(
- std::move(st.TicketsWithErrors),
- gotTickets);
- if (!c) {
- LogWarning("UpdateMissingServiceTickets: new cache is NULL. BUG?");
- c = cache;
- }
-
- if (!ServiceTicketsFilepath_) {
- return c;
- }
-
- DiskCacheServiceTickets_ = AppendToJsonArray(DiskCacheServiceTickets_, st.Responses);
-
- TDiskWriter w(ServiceTicketsFilepath_, Logger_.Get());
- w.Write(PrepareTicketsForDisk(DiskCacheServiceTickets_, Settings_.GetSelfTvmId()));
-
- return c;
- }
-
- void TThreadedUpdater::UpdatePublicKeys() {
- if (!Settings_.IsServiceTicketCheckingRequired() && !Settings_.IsUserTicketCheckingRequired()) {
- return;
- }
-
- TInstant pkut = GetUpdateTimeOfPublicKeys();
- if (!IsTimeToUpdatePublicKeys(pkut)) {
- return;
- }
-
- try {
- TString publicKeys = GetPublicKeysFromHttp();
-
- UpdatePublicKeysCache(publicKeys, TInstant::Now());
- if (PublicKeysFilepath_) {
- TDiskWriter w(PublicKeysFilepath_, Logger_.Get());
- w.Write(publicKeys);
- }
- if (IsServiceContextOk() && IsUserContextOk()) {
- ClearError(EScope::PublicKeys);
- }
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::PublicKeys, e.what());
- LogWarning(TStringBuilder() << "Failed to update public keys: " << e.what());
- if (TInstant::Now() - pkut > PublicKeysDurations_.Expiring) {
- LogError("Public keys have not been refreshed for too long period");
- }
- }
- }
-
- void TThreadedUpdater::UpdateRoles() {
- if (!RolesFetcher_) {
- return;
- }
-
- TInstant rut = GetUpdateTimeOfRoles();
- if (!TRolesFetcher::IsTimeToUpdate(RetrySettings_, TInstant::Now() - rut)) {
- return;
- }
-
- struct TCloser {
- TRolesFetcher* Fetcher;
- ~TCloser() {
- Fetcher->ResetConnection();
- }
- } closer{RolesFetcher_.get()};
-
- try {
- TServiceTicketsPtr st = GetCachedServiceTickets();
- Y_ENSURE(st, "No one service ticket in memory: how it possible?");
- auto it = st->TicketsById.find(Settings_.GetTiroleTvmId());
- Y_ENSURE(it != st->TicketsById.end(),
- "Missing tvmid for tirole in cache: " << Settings_.GetTiroleTvmId());
-
- RolesFetcher_->Update(
- FetchWithRetries(
- [&]() { return RolesFetcher_->FetchActualRoles(it->second); },
- EScope::Roles));
- SetUpdateTimeOfRoles(TInstant::Now());
-
- if (RolesFetcher_->AreRolesOk()) {
- ClearError(EScope::Roles);
- }
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::Roles, e.what());
- LogWarning(TStringBuilder() << "Failed to update roles: " << e.what());
- if (TRolesFetcher::ShouldWarn(RetrySettings_, TInstant::Now() - rut)) {
- LogError("Roles have not been refreshed for too long period");
- }
- }
- }
-
- TServiceTicketsPtr TThreadedUpdater::UpdateServiceTicketsCachePartly(
- TAsyncUpdaterBase::TPairTicketsErrors&& tickets,
- size_t got) {
- size_t count = tickets.Tickets.size();
- TServiceTicketsPtr c = MakeIntrusiveConst<TServiceTickets>(std::move(tickets.Tickets),
- std::move(tickets.Errors),
- DstAliases_);
- SetServiceTickets(c);
-
- LogInfo(TStringBuilder()
- << "Cache was partly updated with " << got
- << " service ticket(s). total: " << count);
-
- return c;
- }
-
- void TThreadedUpdater::UpdateServiceTicketsCache(TPairTicketsErrors&& tickets, TInstant time) {
- size_t count = tickets.Tickets.size();
- SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(std::move(tickets.Tickets),
- std::move(tickets.Errors),
- DstAliases_));
-
- SetUpdateTimeOfServiceTickets(time);
-
- if (count > 0) {
- LogInfo(TStringBuilder() << "Cache was updated with " << count << " service ticket(s): " << time);
- }
- }
-
- void TThreadedUpdater::UpdatePublicKeysCache(const TString& publicKeys, TInstant time) {
- if (publicKeys.empty()) {
- return;
- }
-
- if (Settings_.IsServiceTicketCheckingRequired()) {
- SetServiceContext(MakeIntrusiveConst<TServiceContext>(
- TServiceContext::CheckingFactory(Settings_.GetSelfTvmId(),
- publicKeys)));
- }
-
- if (Settings_.IsUserTicketCheckingRequired()) {
- SetUserContext(publicKeys);
- }
-
- SetUpdateTimeOfPublicKeys(time);
-
- LogInfo(TStringBuilder() << "Cache was updated with public keys: " << time);
- }
-
- void TThreadedUpdater::ReadStateFromDisk() {
- try {
- TServiceTicketsFromDisk st = ReadServiceTicketsFromDisk();
- UpdateServiceTicketsCache(std::move(st.TicketsWithErrors), st.BornDate);
- DiskCacheServiceTickets_ = st.FileBody;
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder() << "Failed to read service tickets from disk: " << e.what());
- }
-
- try {
- std::pair<TString, TInstant> pk = ReadPublicKeysFromDisk();
- UpdatePublicKeysCache(pk.first, pk.second);
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder() << "Failed to read public keys from disk: " << e.what());
- }
-
- try {
- TString rs = ReadRetrySettingsFromDisk();
- UpdateRetrySettings(rs);
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder() << "Failed to read retry settings from disk: " << e.what());
- }
-
- try {
- if (RolesFetcher_) {
- SetUpdateTimeOfRoles(RolesFetcher_->ReadFromDisk());
- }
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder() << "Failed to read roles from disk: " << e.what());
- }
- }
-
- TThreadedUpdater::TServiceTicketsFromDisk TThreadedUpdater::ReadServiceTicketsFromDisk() const {
- if (!ServiceTicketsFilepath_) {
- return {};
- }
-
- TDiskReader r(ServiceTicketsFilepath_, Logger_.Get());
- if (!r.Read()) {
- return {};
- }
-
- std::pair<TStringBuf, TTvmId> data = ParseTicketsFromDisk(r.Data());
- if (data.second != Settings_.GetSelfTvmId()) {
- TStringStream s;
- s << "Disk cache is for another tvmId (" << data.second << "). ";
- s << "Self=" << Settings_.GetSelfTvmId();
- LogWarning(s.Str());
- return {};
- }
-
- TPairTicketsErrors res;
- ParseTicketsFromResponse(data.first, Destinations_, res);
- if (IsInvalid(TServiceTickets::GetInvalidationTime(res.Tickets), TInstant::Now())) {
- LogWarning("Disk cache (service tickets) is too old");
- return {};
- }
-
- LogInfo(TStringBuilder() << "Got " << res.Tickets.size() << " service ticket(s) from disk");
- return {std::move(res), r.Time(), TString(data.first)};
- }
-
- std::pair<TString, TInstant> TThreadedUpdater::ReadPublicKeysFromDisk() const {
- if (!PublicKeysFilepath_) {
- return {};
- }
-
- TDiskReader r(PublicKeysFilepath_, Logger_.Get());
- if (!r.Read()) {
- return {};
- }
-
- if (TInstant::Now() - r.Time() > PublicKeysDurations_.Invalid) {
- LogWarning("Disk cache (public keys) is too old");
- return {};
- }
-
- return {r.Data(), r.Time()};
- }
-
- TString TThreadedUpdater::ReadRetrySettingsFromDisk() const {
- if (!RetrySettingsFilepath_) {
- return {};
- }
-
- TDiskReader r(RetrySettingsFilepath_, Logger_.Get());
- if (!r.Read()) {
- return {};
- }
-
- return r.Data();
- }
-
- template <class Dsts>
- TThreadedUpdater::THttpResult TThreadedUpdater::GetServiceTicketsFromHttp(const Dsts& dsts, const size_t dstLimit) const {
- Y_ENSURE(SigningContext_, "Internal error");
-
- TClientSettings::TDstVector part;
- part.reserve(dstLimit);
- THttpResult res;
- res.TicketsWithErrors.Tickets.reserve(dsts.size());
- res.Responses.reserve(dsts.size() / dstLimit + 1);
-
- for (auto it = dsts.begin(); it != dsts.end();) {
- part.clear();
- for (size_t count = 0; it != dsts.end() && count < dstLimit; ++count, ++it) {
- part.push_back(*it);
- }
-
- TString response =
- FetchWithRetries(
- [this, &part]() {
- // create request here to keep 'ts' actual
- return FetchServiceTicketsFromHttp(PrepareRequestForServiceTickets(
- Settings_.GetSelfTvmId(),
- *SigningContext_,
- part,
- ProcInfo_));
- },
- EScope::ServiceTickets)
- .Response;
- ParseTicketsFromResponse(response, part, res.TicketsWithErrors);
- LogDebug(TStringBuilder()
- << "Response with service tickets for " << part.size()
- << " destination(s) was successfully fetched from " << TvmUrl_);
-
- res.Responses.push_back(response);
- }
-
- LogDebug(TStringBuilder()
- << "Got responses with service tickets with " << res.Responses.size() << " pages for "
- << dsts.size() << " destination(s)");
- for (const auto& p : res.TicketsWithErrors.Errors) {
- LogError(TStringBuilder()
- << "Failed to get service ticket for dst=" << p.first << ": " << p.second);
- }
-
- return res;
- }
-
- TString TThreadedUpdater::GetPublicKeysFromHttp() const {
- TString publicKeys =
- FetchWithRetries(
- [this]() { return FetchPublicKeysFromHttp(); },
- EScope::PublicKeys)
- .Response;
-
- LogDebug("Public keys were successfully fetched from " + TvmUrl_);
-
- return publicKeys;
- }
-
- NUtils::TFetchResult TThreadedUpdater::FetchServiceTicketsFromHttp(const TString& body) const {
- TStringStream s;
-
- THttpHeaders outHeaders;
- TKeepAliveHttpClient::THttpCode code = GetClient().DoPost("/2/ticket", body, &s, Headers_, &outHeaders);
-
- const THttpInputHeader* settings = outHeaders.FindHeader("X-Ya-Retry-Settings");
-
- return {code, {}, "/2/ticket", s.Str(), settings ? settings->Value() : ""};
- }
-
- NUtils::TFetchResult TThreadedUpdater::FetchPublicKeysFromHttp() const {
- TStringStream s;
-
- THttpHeaders outHeaders;
- TKeepAliveHttpClient::THttpCode code = GetClient().DoGet(PublicKeysUrl_, &s, {}, &outHeaders);
-
- const THttpInputHeader* settings = outHeaders.FindHeader("X-Ya-Retry-Settings");
-
- return {code, {}, "/2/keys", s.Str(), settings ? settings->Value() : ""};
- }
-
- bool TThreadedUpdater::UpdateRetrySettings(const TString& header) const {
- if (header.empty()) {
- // Probably it is some kind of test?
- return false;
- }
-
- try {
- TString raw = NUtils::Base64url2bin(header);
- Y_ENSURE(raw, "Invalid base64url in settings");
-
- retry_settings::v1::Settings proto;
- Y_ENSURE(proto.ParseFromString(raw), "Invalid proto");
-
- // This ugly hack helps to process these settings in any case
- TThreadedUpdater& this_ = *const_cast<TThreadedUpdater*>(this);
- TRetrySettings& res = this_.RetrySettings_;
-
- TStringStream diff;
- auto update = [&diff](auto& l, const auto& r, TStringBuf desc) {
- if (l != r) {
- diff << desc << ":" << l << "->" << r << ";";
- l = r;
- }
- };
-
- if (proto.has_exponential_backoff_min_sec()) {
- update(res.BackoffSettings.Min,
- TDuration::Seconds(proto.exponential_backoff_min_sec()),
- "exponential_backoff_min");
- }
- if (proto.has_exponential_backoff_max_sec()) {
- update(res.BackoffSettings.Max,
- TDuration::Seconds(proto.exponential_backoff_max_sec()),
- "exponential_backoff_max");
- }
- if (proto.has_exponential_backoff_factor()) {
- update(res.BackoffSettings.Factor,
- proto.exponential_backoff_factor(),
- "exponential_backoff_factor");
- }
- if (proto.has_exponential_backoff_jitter()) {
- update(res.BackoffSettings.Jitter,
- proto.exponential_backoff_jitter(),
- "exponential_backoff_jitter");
- }
- this_.ExpBackoff_.UpdateSettings(res.BackoffSettings);
-
- if (proto.has_max_random_sleep_default()) {
- update(res.MaxRandomSleepDefault,
- TDuration::MilliSeconds(proto.max_random_sleep_default()),
- "max_random_sleep_default");
- }
- if (proto.has_max_random_sleep_when_ok()) {
- update(res.MaxRandomSleepWhenOk,
- TDuration::MilliSeconds(proto.max_random_sleep_when_ok()),
- "max_random_sleep_when_ok");
- }
- if (proto.has_retries_on_start()) {
- Y_ENSURE(proto.retries_on_start(), "retries_on_start==0");
- update(res.RetriesOnStart,
- proto.retries_on_start(),
- "retries_on_start");
- }
- if (proto.has_retries_in_background()) {
- Y_ENSURE(proto.retries_in_background(), "retries_in_background==0");
- update(res.RetriesInBackground,
- proto.retries_in_background(),
- "retries_in_background");
- }
- if (proto.has_worker_awaking_period_sec()) {
- update(res.WorkerAwakingPeriod,
- TDuration::Seconds(proto.worker_awaking_period_sec()),
- "worker_awaking_period");
- this_.WorkerAwakingPeriod_ = res.WorkerAwakingPeriod;
- }
- if (proto.has_dsts_limit()) {
- Y_ENSURE(proto.dsts_limit(), "dsts_limit==0");
- update(res.DstsLimit,
- proto.dsts_limit(),
- "dsts_limit");
- }
-
- if (proto.has_roles_update_period_sec()) {
- Y_ENSURE(proto.roles_update_period_sec(), "roles_update_period==0");
- update(res.RolesUpdatePeriod,
- TDuration::Seconds(proto.roles_update_period_sec()),
- "roles_update_period_sec");
- }
- if (proto.has_roles_warn_period_sec()) {
- Y_ENSURE(proto.roles_warn_period_sec(), "roles_warn_period_sec==0");
- update(res.RolesWarnPeriod,
- TDuration::Seconds(proto.roles_warn_period_sec()),
- "roles_warn_period_sec");
- }
-
- if (diff.empty()) {
- return false;
- }
-
- LogDebug("Retry settings were updated: " + diff.Str());
- return true;
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder()
- << "Failed to update retry settings from server, header '"
- << header << "': "
- << e.what());
- }
-
- return false;
- }
-
- template <typename Func>
- NUtils::TFetchResult TThreadedUpdater::FetchWithRetries(Func func, EScope scope) const {
- const ui32 tries = Inited_ ? RetrySettings_.RetriesInBackground
- : RetrySettings_.RetriesOnStart;
-
- for (size_t idx = 1;; ++idx) {
- RandomSleep();
-
- try {
- NUtils::TFetchResult result = func();
-
- if (UpdateRetrySettings(result.RetrySettings) && RetrySettingsFilepath_) {
- TDiskWriter w(RetrySettingsFilepath_, Logger_.Get());
- w.Write(result.RetrySettings);
- }
-
- if (400 <= result.Code && result.Code <= 499) {
- throw TNonRetriableException() << ProcessHttpError(scope, result.Path, result.Code, result.Response);
- }
- if (result.Code < 200 || result.Code >= 399) {
- throw yexception() << ProcessHttpError(scope, result.Path, result.Code, result.Response);
- }
-
- ExpBackoff_.Decrease();
- return result;
- } catch (const TNonRetriableException& e) {
- LogWarning(TStringBuilder() << "Failed to get " << scope << ": " << e.what());
- ExpBackoff_.Increase();
- throw;
- } catch (const std::exception& e) {
- LogWarning(TStringBuilder() << "Failed to get " << scope << ": " << e.what());
- ExpBackoff_.Increase();
- if (idx >= tries) {
- throw;
- }
- }
- }
-
- throw yexception() << "unreachable";
- }
-
- void TThreadedUpdater::RandomSleep() const {
- const TDuration maxSleep = TClientStatus::ECode::Ok == GetState()
- ? RetrySettings_.MaxRandomSleepWhenOk
- : RetrySettings_.MaxRandomSleepDefault;
-
- if (maxSleep) {
- ui32 toSleep = Random_.GenRand() % maxSleep.MilliSeconds();
- ExpBackoff_.Sleep(TDuration::MilliSeconds(toSleep));
- }
- }
-
- TString TThreadedUpdater::PrepareRequestForServiceTickets(TTvmId src,
- const TServiceContext& ctx,
- const TClientSettings::TDstVector& dsts,
- const NUtils::TProcInfo& procInfo,
- time_t now) {
- TStringStream s;
-
- const TString ts = IntToString<10>(now);
- TStringStream dst;
- dst.Reserve(10 * dsts.size());
- for (const TClientSettings::TDst& d : dsts) {
- if (dst.Str()) {
- dst << ',';
- }
- dst << d.Id;
- }
-
- s << "grant_type=client_credentials";
- s << "&src=" << src;
- s << "&dst=" << dst.Str();
- s << "&ts=" << ts;
- s << "&sign=" << ctx.SignCgiParamsForTvm(ts, dst.Str());
- s << "&get_retry_settings=yes";
-
- s << "&";
- procInfo.AddToRequest(s);
-
- return s.Str();
- }
-
- template <class Dsts>
- void TThreadedUpdater::ParseTicketsFromResponse(TStringBuf resp,
- const Dsts& dsts,
- TPairTicketsErrors& out) const {
- NJson::TJsonValue doc;
- Y_ENSURE(NJson::ReadJsonTree(resp, &doc), "Invalid json from tvm-api: " << resp);
-
- const NJson::TJsonValue* currentResp = doc.IsMap() ? &doc : nullptr;
- auto find = [&currentResp, &doc](TTvmId id, NJson::TJsonValue& obj) -> bool {
- const TString idStr = IntToString<10>(id);
- if (currentResp && currentResp->GetValue(idStr, &obj)) {
- return true;
- }
-
- for (const NJson::TJsonValue& val : doc.GetArray()) {
- currentResp = &val;
- if (currentResp->GetValue(idStr, &obj)) {
- return true;
- }
- }
-
- return false;
- };
-
- for (const TClientSettings::TDst& d : dsts) {
- NJson::TJsonValue obj;
- NJson::TJsonValue val;
-
- if (!find(d.Id, obj) || !obj.GetValue("ticket", &val)) {
- TString err;
- if (obj.GetValue("error", &val)) {
- err = val.GetString();
- } else {
- err = "Missing tvm_id in response, should never happend: " + IntToString<10>(d.Id);
- }
-
- TStringStream s;
- s << "Failed to get ServiceTicket for " << d.Id << ": " << err;
- ProcessError(EType::NonRetriable, EScope::ServiceTickets, s.Str());
-
- out.Errors.insert({d.Id, std::move(err)});
- continue;
- }
-
- out.Tickets.insert({d.Id, val.GetString()});
- }
- }
-
- static const char DELIMETER = '\t';
- TString TThreadedUpdater::PrepareTicketsForDisk(TStringBuf tvmResponse, TTvmId selfId) {
- TStringStream s;
- s << tvmResponse << DELIMETER << selfId;
- return s.Str();
- }
-
- std::pair<TStringBuf, TTvmId> TThreadedUpdater::ParseTicketsFromDisk(TStringBuf data) {
- TStringBuf tvmId = data.RNextTok(DELIMETER);
- return {data, IntFromString<TTvmId, 10>(tvmId)};
- }
-
- const TDstSet& TThreadedUpdater::GetDsts() const {
- return Destinations_;
- }
-
- void TThreadedUpdater::AddDstToSettings(const TClientSettings::TDst& dst) {
- Destinations_.insert(dst);
- }
-
- bool TThreadedUpdater::IsTimeToUpdateServiceTickets(TInstant lastUpdate) const {
- return TInstant::Now() - lastUpdate > ServiceTicketsDurations_.RefreshPeriod;
- }
-
- bool TThreadedUpdater::IsTimeToUpdatePublicKeys(TInstant lastUpdate) const {
- return TInstant::Now() - lastUpdate > PublicKeysDurations_.RefreshPeriod;
- }
-
- bool TThreadedUpdater::AreServicesTicketsOk() const {
- if (!Settings_.IsServiceTicketFetchingRequired()) {
- return true;
- }
- auto c = GetCachedServiceTickets();
- return c && (!Settings_.IsIncompleteTicketsSetAnError || c->TicketsById.size() == Destinations_.size());
- }
-
- bool TThreadedUpdater::IsServiceContextOk() const {
- if (!Settings_.IsServiceTicketCheckingRequired()) {
- return true;
- }
-
- return bool(GetCachedServiceContext());
- }
-
- bool TThreadedUpdater::IsUserContextOk() const {
- if (!Settings_.IsUserTicketCheckingRequired()) {
- return true;
- }
- return bool(GetCachedUserContext());
- }
-
- void TThreadedUpdater::Worker() {
- UpdateServiceTickets();
- UpdatePublicKeys();
- UpdateRoles();
- }
-
- TServiceTickets::TMapAliasId TThreadedUpdater::MakeAliasMap(const TClientSettings& settings) {
- TServiceTickets::TMapAliasId res;
-
- if (settings.HasDstAliases()) {
- for (const auto& pair : settings.GetDstAliases()) {
- res.insert({pair.first, pair.second.Id});
- }
- }
-
- return res;
- }
-
- TClientSettings::TDstVector TThreadedUpdater::FindMissingDsts(TServiceTicketsPtr available, const TDstSet& required) {
- Y_ENSURE(available);
- TDstSet set;
- // available->TicketsById is not sorted
- for (const auto& pair : available->TicketsById) {
- set.insert(pair.first);
- }
- return FindMissingDsts(set, required);
- }
-
- TClientSettings::TDstVector TThreadedUpdater::FindMissingDsts(const TDstSet& available, const TDstSet& required) {
- TClientSettings::TDstVector res;
- std::set_difference(required.begin(), required.end(),
- available.begin(), available.end(),
- std::inserter(res, res.begin()));
- return res;
- }
-
- TString TThreadedUpdater::CreateJsonArray(const TSmallVec<TString>& responses) {
- if (responses.empty()) {
- return "[]";
- }
-
- size_t size = 0;
- for (const TString& r : responses) {
- size += r.size() + 1;
- }
-
- TString res;
- res.reserve(size + 2);
-
- res.push_back('[');
- for (const TString& r : responses) {
- res.append(r).push_back(',');
- }
- res.back() = ']';
-
- return res;
- }
-
- TString TThreadedUpdater::AppendToJsonArray(const TString& json, const TSmallVec<TString>& responses) {
- Y_ENSURE(json, "previous body required");
-
- size_t size = 0;
- for (const TString& r : responses) {
- size += r.size() + 1;
- }
-
- TString res;
- res.reserve(size + 2 + json.size());
-
- res.push_back('[');
- if (json.StartsWith('[')) {
- Y_ENSURE(json.EndsWith(']'), "array is broken:" << json);
- res.append(TStringBuf(json).Chop(1).Skip(1));
- } else {
- res.append(json);
- }
-
- res.push_back(',');
- for (const TString& r : responses) {
- res.append(r).push_back(',');
- }
- res.back() = ']';
-
- return res;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/api/threaded_updater.h b/library/cpp/tvmauth/client/misc/api/threaded_updater.h
deleted file mode 100644
index e546bbe030..0000000000
--- a/library/cpp/tvmauth/client/misc/api/threaded_updater.h
+++ /dev/null
@@ -1,140 +0,0 @@
-#pragma once
-
-#include "retry_settings.h"
-#include "roles_fetcher.h"
-#include "settings.h"
-
-#include <library/cpp/tvmauth/client/misc/async_updater.h>
-#include <library/cpp/tvmauth/client/misc/threaded_updater.h>
-
-#include <util/generic/set.h>
-#include <util/random/fast.h>
-
-namespace NTvmAuth::NTvmApi {
- using TDstSet = TSet<TClientSettings::TDst>;
-
- class TThreadedUpdater: public TThreadedUpdaterBase {
- public:
- /*!
- * Starts thread for updating of in-memory cache in background
- * Reads cache from disk if specified
- * @param settings
- * @param logger is usefull for monitoring and debuging
- */
- static TAsyncUpdaterPtr Create(const TClientSettings& settings, TLoggerPtr logger);
- ~TThreadedUpdater();
-
- TClientStatus GetStatus() const override;
- NRoles::TRolesPtr GetRoles() const override;
-
- protected: // for tests
- TClientStatus::ECode GetState() const;
-
- TThreadedUpdater(const TClientSettings& settings, TLoggerPtr logger);
- void Init();
-
- void UpdateServiceTickets();
- void UpdateAllServiceTickets();
- TServiceTicketsPtr UpdateMissingServiceTickets(const TDstSet& required);
- void UpdatePublicKeys();
- void UpdateRoles();
-
- TServiceTicketsPtr UpdateServiceTicketsCachePartly(TPairTicketsErrors&& tickets, size_t got);
- void UpdateServiceTicketsCache(TPairTicketsErrors&& tickets, TInstant time);
- void UpdatePublicKeysCache(const TString& publicKeys, TInstant time);
-
- void ReadStateFromDisk();
-
- struct TServiceTicketsFromDisk {
- TPairTicketsErrors TicketsWithErrors;
- TInstant BornDate;
- TString FileBody;
- };
-
- TServiceTicketsFromDisk ReadServiceTicketsFromDisk() const;
- std::pair<TString, TInstant> ReadPublicKeysFromDisk() const;
- TString ReadRetrySettingsFromDisk() const;
-
- struct THttpResult {
- TPairTicketsErrors TicketsWithErrors;
- TSmallVec<TString> Responses;
- };
-
- template <class Dsts>
- THttpResult GetServiceTicketsFromHttp(const Dsts& dsts, const size_t dstLimit) const;
- TString GetPublicKeysFromHttp() const;
-
- virtual NUtils::TFetchResult FetchServiceTicketsFromHttp(const TString& body) const;
- virtual NUtils::TFetchResult FetchPublicKeysFromHttp() const;
-
- bool UpdateRetrySettings(const TString& header) const;
-
- template <typename Func>
- NUtils::TFetchResult FetchWithRetries(Func func, EScope scope) const;
- void RandomSleep() const;
-
- static TString PrepareRequestForServiceTickets(TTvmId src,
- const TServiceContext& ctx,
- const TClientSettings::TDstVector& dsts,
- const NUtils::TProcInfo& procInfo,
- time_t now = time(nullptr));
- template <class Dsts>
- void ParseTicketsFromResponse(TStringBuf resp,
- const Dsts& dsts,
- TPairTicketsErrors& out) const;
-
- static TString PrepareTicketsForDisk(TStringBuf tvmResponse, TTvmId selfId);
- static std::pair<TStringBuf, TTvmId> ParseTicketsFromDisk(TStringBuf data);
-
- const TDstSet& GetDsts() const;
- void AddDstToSettings(const TClientSettings::TDst& dst);
-
- bool IsTimeToUpdateServiceTickets(TInstant lastUpdate) const;
- bool IsTimeToUpdatePublicKeys(TInstant lastUpdate) const;
-
- bool AreServicesTicketsOk() const;
- bool IsServiceContextOk() const;
- bool IsUserContextOk() const;
-
- void Worker() override;
-
- static TServiceTickets::TMapAliasId MakeAliasMap(const TClientSettings& settings);
- static TClientSettings::TDstVector FindMissingDsts(TServiceTicketsPtr available, const TDstSet& required);
- static TClientSettings::TDstVector FindMissingDsts(const TDstSet& available, const TDstSet& required);
-
- static TString CreateJsonArray(const TSmallVec<TString>& responses);
- static TString AppendToJsonArray(const TString& json, const TSmallVec<TString>& responses);
-
- private:
- TRetrySettings RetrySettings_;
-
- protected:
- mutable TExponentialBackoff ExpBackoff_;
-
- private:
- const TClientSettings Settings_;
-
- const NUtils::TProcInfo ProcInfo_;
-
- const TString PublicKeysUrl_;
-
- const TServiceTickets::TMapAliasId DstAliases_;
-
- const TKeepAliveHttpClient::THeaders Headers_;
- TMaybe<TServiceContext> SigningContext_;
-
- TDstSet Destinations_;
- TString DiskCacheServiceTickets_;
- bool NeedFetchMissingServiceTickets_ = true;
-
- TString PublicKeysFilepath_;
- TString ServiceTicketsFilepath_;
- TString RetrySettingsFilepath_;
-
- std::unique_ptr<TRolesFetcher> RolesFetcher_;
-
- mutable TReallyFastRng32 Random_;
-
- bool Inited_ = false;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/async_updater.cpp b/library/cpp/tvmauth/client/misc/async_updater.cpp
deleted file mode 100644
index 9cb0332ed4..0000000000
--- a/library/cpp/tvmauth/client/misc/async_updater.cpp
+++ /dev/null
@@ -1,152 +0,0 @@
-#include "async_updater.h"
-
-#include "utils.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <util/string/builder.h>
-#include <util/system/spin_wait.h>
-
-namespace NTvmAuth {
- TAsyncUpdaterBase::TAsyncUpdaterBase() {
- ServiceTicketsDurations_.RefreshPeriod = TDuration::Hours(1);
- ServiceTicketsDurations_.Expiring = TDuration::Hours(2);
- ServiceTicketsDurations_.Invalid = TDuration::Hours(11);
-
- PublicKeysDurations_.RefreshPeriod = TDuration::Days(1);
- PublicKeysDurations_.Expiring = TDuration::Days(2);
- PublicKeysDurations_.Invalid = TDuration::Days(6);
- }
-
- NRoles::TRolesPtr TAsyncUpdaterBase::GetRoles() const {
- ythrow TIllegalUsage() << "not implemented";
- }
-
- TInstant TAsyncUpdaterBase::GetUpdateTimeOfPublicKeys() const {
- return PublicKeysTime_.Get();
- }
-
- TInstant TAsyncUpdaterBase::GetUpdateTimeOfServiceTickets() const {
- return ServiceTicketsTime_.Get();
- }
-
- TInstant TAsyncUpdaterBase::GetUpdateTimeOfRoles() const {
- return RolesTime_.Get();
- }
-
- TInstant TAsyncUpdaterBase::GetInvalidationTimeOfPublicKeys() const {
- TInstant ins = GetUpdateTimeOfPublicKeys();
- return ins == TInstant() ? TInstant() : ins + PublicKeysDurations_.Invalid;
- }
-
- TInstant TAsyncUpdaterBase::GetInvalidationTimeOfServiceTickets() const {
- TServiceTicketsPtr c = GetCachedServiceTickets();
- return c ? c->InvalidationTime : TInstant();
- }
-
- bool TAsyncUpdaterBase::ArePublicKeysInvalid(TInstant now) const {
- return IsInvalid(GetInvalidationTimeOfPublicKeys(), now);
- }
-
- bool TAsyncUpdaterBase::AreServiceTicketsInvalid(TInstant now) const {
- TServiceTicketsPtr c = GetCachedServiceTickets();
- // Empty set of tickets is allways valid.
- return c && !c->TicketsById.empty() && IsInvalid(GetInvalidationTimeOfServiceTickets(), now);
- }
-
- bool TAsyncUpdaterBase::IsInvalid(TInstant invTime, TInstant now) {
- return invTime -
- TDuration::Minutes(1) // lag for closing from balancer
- < now;
- }
-
- void TAsyncUpdaterBase::SetBbEnv(EBlackboxEnv original, TMaybe<EBlackboxEnv> overrided) {
- if (overrided) {
- Y_ENSURE_EX(NUtils::CheckBbEnvOverriding(original, *overrided),
- TBrokenTvmClientSettings() << "Overriding of BlackboxEnv is illegal: "
- << original << " -> " << *overrided);
- }
-
- Envs_.store({original, overrided}, std::memory_order_relaxed);
- }
-
- TServiceTicketsPtr TAsyncUpdaterBase::GetCachedServiceTickets() const {
- return ServiceTickets_.Get();
- }
-
- TServiceContextPtr TAsyncUpdaterBase::GetCachedServiceContext() const {
- return ServiceContext_.Get();
- }
-
- TUserContextPtr TAsyncUpdaterBase::GetCachedUserContext(TMaybe<EBlackboxEnv> overridenEnv) const {
- TAllUserContextsPtr ctx = AllUserContexts_.Get();
- if (!ctx) {
- return nullptr;
- }
-
- const TEnvs envs = Envs_.load(std::memory_order_relaxed);
- if (!envs.Original) {
- return nullptr;
- }
-
- EBlackboxEnv env = *envs.Original;
-
- if (overridenEnv) {
- Y_ENSURE_EX(NUtils::CheckBbEnvOverriding(*envs.Original, *overridenEnv),
- TBrokenTvmClientSettings() << "Overriding of BlackboxEnv is illegal: "
- << *envs.Original << " -> " << *overridenEnv);
- env = *overridenEnv;
- } else if (envs.Overrided) {
- env = *envs.Overrided;
- }
-
- return ctx->Get(env);
- }
-
- void TAsyncUpdaterBase::SetServiceTickets(TServiceTicketsPtr c) {
- ServiceTickets_.Set(std::move(c));
- }
-
- void TAsyncUpdaterBase::SetServiceContext(TServiceContextPtr c) {
- ServiceContext_.Set(std::move(c));
- }
-
- void TAsyncUpdaterBase::SetUserContext(TStringBuf publicKeys) {
- AllUserContexts_.Set(MakeIntrusiveConst<TAllUserContexts>(publicKeys));
- }
-
- void TAsyncUpdaterBase::SetUpdateTimeOfPublicKeys(TInstant ins) {
- PublicKeysTime_.Set(ins);
- }
-
- void TAsyncUpdaterBase::SetUpdateTimeOfServiceTickets(TInstant ins) {
- ServiceTicketsTime_.Set(ins);
- }
-
- void TAsyncUpdaterBase::SetUpdateTimeOfRoles(TInstant ins) {
- RolesTime_.Set(ins);
- }
-
- bool TAsyncUpdaterBase::IsServiceTicketMapOk(TServiceTicketsPtr c, size_t expectedTicketCount, bool strict) {
- return c &&
- (strict
- ? c->TicketsById.size() == expectedTicketCount
- : !c->TicketsById.empty());
- }
-
- TAllUserContexts::TAllUserContexts(TStringBuf publicKeys) {
- auto add = [&, this](EBlackboxEnv env) {
- Ctx_[(size_t)env] = MakeIntrusiveConst<TUserContext>(env, publicKeys);
- };
-
- add(EBlackboxEnv::Prod);
- add(EBlackboxEnv::Test);
- add(EBlackboxEnv::ProdYateam);
- add(EBlackboxEnv::TestYateam);
- add(EBlackboxEnv::Stress);
- }
-
- TUserContextPtr TAllUserContexts::Get(EBlackboxEnv env) const {
- return Ctx_[(size_t)env];
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/async_updater.h b/library/cpp/tvmauth/client/misc/async_updater.h
deleted file mode 100644
index 7b556d7a38..0000000000
--- a/library/cpp/tvmauth/client/misc/async_updater.h
+++ /dev/null
@@ -1,183 +0,0 @@
-#pragma once
-
-#include "last_error.h"
-#include "settings.h"
-#include "roles/roles.h"
-
-#include <library/cpp/tvmauth/client/client_status.h>
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/tvmauth/deprecated/service_context.h>
-#include <library/cpp/tvmauth/deprecated/user_context.h>
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/hash.h>
-#include <util/generic/maybe.h>
-#include <util/generic/noncopyable.h>
-#include <util/generic/ptr.h>
-
-#include <array>
-#include <atomic>
-
-namespace NTvmAuth::NInternal {
- class TClientCaningKnife;
-}
-
-namespace NTvmAuth {
- class TServiceTickets: public TAtomicRefCount<TServiceTickets> {
- public:
- using TMapAliasStr = THashMap<TClientSettings::TAlias, TString>;
- using TMapIdStr = THashMap<TTvmId, TString>;
- using TIdSet = THashSet<TTvmId>;
- using TAliasSet = THashSet<TClientSettings::TAlias>;
- using TMapAliasId = THashMap<TClientSettings::TAlias, TTvmId>;
-
- TServiceTickets(TMapIdStr&& tickets, TMapIdStr&& errors, const TMapAliasId& dstMap)
- : TicketsById(std::move(tickets))
- , ErrorsById(std::move(errors))
- {
- InitAliasesAndUnfetchedIds(dstMap);
- InitInvalidationTime();
- }
-
- static TInstant GetInvalidationTime(const TMapIdStr& ticketsById) {
- TInstant res;
-
- for (const auto& pair : ticketsById) {
- TMaybe<TInstant> t = NTvmAuth::NInternal::TCanningKnife::GetExpirationTime(pair.second);
- if (!t) {
- continue;
- }
-
- res = res == TInstant() ? *t : std::min(res, *t);
- }
-
- return res;
- }
-
- public:
- TMapIdStr TicketsById;
- TMapIdStr ErrorsById;
- TMapAliasStr TicketsByAlias;
- TMapAliasStr ErrorsByAlias;
- TInstant InvalidationTime;
- TIdSet UnfetchedIds;
- TAliasSet UnfetchedAliases;
-
- private:
- void InitAliasesAndUnfetchedIds(const TMapAliasId& dstMap) {
- for (const auto& pair : dstMap) {
- auto it = TicketsById.find(pair.second);
- auto errIt = ErrorsById.find(pair.second);
-
- if (it == TicketsById.end()) {
- if (errIt != ErrorsById.end()) {
- Y_ENSURE(ErrorsByAlias.insert({pair.first, errIt->second}).second,
- "failed to add: " << pair.first);
- } else {
- UnfetchedAliases.insert(pair.first);
- UnfetchedIds.insert(pair.second);
- }
- } else {
- Y_ENSURE(TicketsByAlias.insert({pair.first, it->second}).second,
- "failed to add: " << pair.first);
- }
- }
- }
-
- void InitInvalidationTime() {
- InvalidationTime = GetInvalidationTime(TicketsById);
- }
- };
- using TServiceTicketsPtr = TIntrusiveConstPtr<TServiceTickets>;
-
- class TAllUserContexts: public TAtomicRefCount<TAllUserContexts> {
- public:
- TAllUserContexts(TStringBuf publicKeys);
-
- TUserContextPtr Get(EBlackboxEnv env) const;
-
- private:
- std::array<TUserContextPtr, 5> Ctx_;
- };
- using TAllUserContextsPtr = TIntrusiveConstPtr<TAllUserContexts>;
-
- class TAsyncUpdaterBase: public TAtomicRefCount<TAsyncUpdaterBase>, protected TLastError, TNonCopyable {
- public:
- TAsyncUpdaterBase();
- virtual ~TAsyncUpdaterBase() = default;
-
- virtual TClientStatus GetStatus() const = 0;
- virtual NRoles::TRolesPtr GetRoles() const;
-
- TServiceTicketsPtr GetCachedServiceTickets() const;
- TServiceContextPtr GetCachedServiceContext() const;
- TUserContextPtr GetCachedUserContext(TMaybe<EBlackboxEnv> overridenEnv = {}) const;
-
- TInstant GetUpdateTimeOfPublicKeys() const;
- TInstant GetUpdateTimeOfServiceTickets() const;
- TInstant GetUpdateTimeOfRoles() const;
- TInstant GetInvalidationTimeOfPublicKeys() const;
- TInstant GetInvalidationTimeOfServiceTickets() const;
-
- bool ArePublicKeysInvalid(TInstant now) const;
- bool AreServiceTicketsInvalid(TInstant now) const;
- static bool IsInvalid(TInstant invTime, TInstant now);
-
- protected:
- void SetBbEnv(EBlackboxEnv original, TMaybe<EBlackboxEnv> overrided = {});
-
- void SetServiceTickets(TServiceTicketsPtr c);
- void SetServiceContext(TServiceContextPtr c);
- void SetUserContext(TStringBuf publicKeys);
- void SetUpdateTimeOfPublicKeys(TInstant ins);
- void SetUpdateTimeOfServiceTickets(TInstant ins);
- void SetUpdateTimeOfRoles(TInstant ins);
-
- static bool IsServiceTicketMapOk(TServiceTicketsPtr c, size_t expectedTicketCount, bool strict);
-
- protected:
- struct TPairTicketsErrors {
- TServiceTickets::TMapIdStr Tickets;
- TServiceTickets::TMapIdStr Errors;
-
- bool operator==(const TPairTicketsErrors& o) const {
- return Tickets == o.Tickets && Errors == o.Errors;
- }
- };
-
- struct TStateDurations {
- TDuration RefreshPeriod;
- TDuration Expiring;
- TDuration Invalid;
- };
-
- TStateDurations ServiceTicketsDurations_;
- TStateDurations PublicKeysDurations_;
-
- protected:
- virtual void StartTvmClientStopping() const {
- }
- virtual bool IsTvmClientStopped() const {
- return true;
- }
- friend class NTvmAuth::NInternal::TClientCaningKnife;
-
- private:
- struct TEnvs {
- TMaybe<EBlackboxEnv> Original;
- TMaybe<EBlackboxEnv> Overrided;
- };
- static_assert(sizeof(TEnvs) <= 8, "Small struct is easy to store as atomic");
- std::atomic<TEnvs> Envs_ = {{}};
-
- NUtils::TProtectedValue<TServiceTicketsPtr> ServiceTickets_;
- NUtils::TProtectedValue<TServiceContextPtr> ServiceContext_;
- NUtils::TProtectedValue<TAllUserContextsPtr> AllUserContexts_;
- NUtils::TProtectedValue<TInstant> PublicKeysTime_;
- NUtils::TProtectedValue<TInstant> ServiceTicketsTime_;
- NUtils::TProtectedValue<TInstant> RolesTime_;
- };
- using TAsyncUpdaterPtr = TIntrusiveConstPtr<TAsyncUpdaterBase>;
-}
diff --git a/library/cpp/tvmauth/client/misc/checker.h b/library/cpp/tvmauth/client/misc/checker.h
deleted file mode 100644
index e8ed2f5503..0000000000
--- a/library/cpp/tvmauth/client/misc/checker.h
+++ /dev/null
@@ -1,67 +0,0 @@
-#pragma once
-
-#include "async_updater.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-namespace NTvmAuth {
- class TServiceTicketChecker {
- public:
- TServiceTicketChecker(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- Y_ENSURE(Updater_);
- GetCache();
- }
-
- /*!
- * Checking must be enabled in TClientSettings
- * Can throw exception if cache is out of date or wrong config
- * @param ticket
- */
- TCheckedServiceTicket Check(TStringBuf ticket) const {
- return GetCache()->Check(ticket);
- }
-
- private:
- TServiceContextPtr GetCache() const {
- TServiceContextPtr c = Updater_->GetCachedServiceContext();
- Y_ENSURE_EX(c, TBrokenTvmClientSettings() << "Need to use TClientSettings::EnableServiceTicketChecking()");
- return c;
- }
-
- private:
- TAsyncUpdaterPtr Updater_;
- };
-
- class TUserTicketChecker {
- public:
- TUserTicketChecker(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- Y_ENSURE(Updater_);
- GetCache({});
- }
-
- /*!
- * Blackbox enviroment must be cofingured in TClientSettings
- * Can throw exception if cache is out of date or wrong config
- */
- TCheckedUserTicket Check(TStringBuf ticket, TMaybe<EBlackboxEnv> overridenEnv) const {
- return GetCache(overridenEnv)->Check(ticket);
- }
-
- private:
- TUserContextPtr GetCache(TMaybe<EBlackboxEnv> overridenEnv) const {
- TUserContextPtr c = Updater_->GetCachedUserContext(overridenEnv);
- Y_ENSURE_EX(c, TBrokenTvmClientSettings() << "Need to use TClientSettings::EnableUserTicketChecking()");
- return c;
- }
-
- private:
- TAsyncUpdaterPtr Updater_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/default_uid_checker.h b/library/cpp/tvmauth/client/misc/default_uid_checker.h
deleted file mode 100644
index 1594f826bd..0000000000
--- a/library/cpp/tvmauth/client/misc/default_uid_checker.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#pragma once
-
-#include "async_updater.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-#include <library/cpp/tvmauth/src/user_impl.h>
-
-namespace NTvmAuth {
- class TDefaultUidChecker {
- public:
- TDefaultUidChecker(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- Y_ENSURE(Updater_);
- GetCache();
- }
-
- /*!
- * Checking must be enabled in TClientSettings
- * Can throw exception if cache is out of date or wrong config
- * @param ticket
- */
- TCheckedUserTicket Check(TCheckedUserTicket ticket) const {
- NRoles::TConsumerRolesPtr roles = GetCache()->GetRolesForUser(ticket);
- if (roles) {
- return ticket;
- }
-
- TUserTicketImplPtr impl = THolder(NInternal::TCanningKnife::GetU(ticket));
- impl->SetStatus(ETicketStatus::NoRoles);
- return TCheckedUserTicket(std::move(impl));
- }
-
- private:
- NRoles::TRolesPtr GetCache() const {
- NRoles::TRolesPtr c = Updater_->GetRoles();
- Y_ENSURE_EX(c, TBrokenTvmClientSettings() << "Need to use TClientSettings::EnableRolesFetching()");
- return c;
- }
-
- private:
- TAsyncUpdaterPtr Updater_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/disk_cache.cpp b/library/cpp/tvmauth/client/misc/disk_cache.cpp
deleted file mode 100644
index 3c01be4a83..0000000000
--- a/library/cpp/tvmauth/client/misc/disk_cache.cpp
+++ /dev/null
@@ -1,165 +0,0 @@
-#include "disk_cache.h"
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <contrib/libs/openssl/include/openssl/evp.h>
-#include <contrib/libs/openssl/include/openssl/hmac.h>
-#include <contrib/libs/openssl/include/openssl/sha.h>
-
-#include <util/folder/path.h>
-#include <util/stream/file.h>
-#include <util/stream/str.h>
-#include <util/system/fs.h>
-#include <util/system/sysstat.h>
-#include <util/system/tempfile.h>
-
-#include <exception>
-
-namespace NTvmAuth {
- static const size_t HASH_SIZE = 32;
- static const size_t TIMESTAMP_SIZE = sizeof(time_t);
-
- TDiskReader::TDiskReader(const TString& filename, ILogger* logger)
- : Filename_(filename)
- , Logger_(logger)
- {
- }
-
- bool TDiskReader::Read() {
- TStringStream s;
-
- try {
- if (!NFs::Exists(Filename_)) {
- if (Logger_) {
- s << "File '" << Filename_ << "' does not exist";
- Logger_->Debug(s.Str());
- }
- return false;
- }
-
- TFile file(Filename_, OpenExisting | RdOnly | Seq);
- file.Flock(LOCK_SH | LOCK_NB);
-
- TFileInput input(file);
- return ParseData(input.ReadAll());
- } catch (const std::exception& e) {
- if (Logger_) {
- s << "Failed to read '" << Filename_ << "': " << e.what();
- Logger_->Error(s.Str());
- }
- }
-
- return false;
- }
-
- bool TDiskReader::ParseData(TStringBuf buf) {
- TStringStream s;
-
- if (buf.size() <= HASH_SIZE + TIMESTAMP_SIZE) {
- if (Logger_) {
- s << "File '" << Filename_ << "' is too small";
- Logger_->Warning(s.Str());
- }
- return false;
- }
-
- TStringBuf hash = buf.SubStr(0, HASH_SIZE);
- if (hash != GetHash(buf.Skip(HASH_SIZE))) {
- if (Logger_) {
- s << "Content of '" << Filename_ << "' was incorrectly changed";
- Logger_->Warning(s.Str());
- }
- return false;
- }
-
- Time_ = TInstant::Seconds(GetTimestamp(buf.substr(0, TIMESTAMP_SIZE)));
- Data_ = buf.Skip(TIMESTAMP_SIZE);
-
- if (Logger_) {
- s << "File '" << Filename_ << "' was successfully read";
- Logger_->Info(s.Str());
- }
- return true;
- }
-
- TString TDiskReader::GetHash(TStringBuf data) {
- TString value(EVP_MAX_MD_SIZE, 0);
- unsigned macLen = 0;
- if (!::HMAC(EVP_sha256(),
- "",
- 0,
- (unsigned char*)data.data(),
- data.size(),
- (unsigned char*)value.data(),
- &macLen)) {
- return {};
- }
-
- if (macLen != EVP_MAX_MD_SIZE) {
- value.resize(macLen);
- }
-
- return value;
- }
-
- time_t TDiskReader::GetTimestamp(TStringBuf data) {
- time_t time = 0;
- for (int idx = TIMESTAMP_SIZE - 1; idx >= 0; --idx) {
- time <<= 8;
- time |= static_cast<unsigned char>(data.at(idx));
- }
- return time;
- }
-
- TDiskWriter::TDiskWriter(const TString& filename, ILogger* logger)
- : Filename_(filename)
- , Logger_(logger)
- {
- }
-
- bool TDiskWriter::Write(TStringBuf data, TInstant now) {
- TStringStream s;
-
- try {
- {
- if (NFs::Exists(Filename_)) {
- Chmod(Filename_.c_str(),
- S_IRUSR | S_IWUSR); // 600
- }
-
- TFile file(Filename_, CreateAlways | WrOnly | Seq | AWUser | ARUser);
- file.Flock(LOCK_EX | LOCK_NB);
-
- TFileOutput output(file);
- output << PrepareData(now, data);
- }
-
- if (Logger_) {
- s << "File '" << Filename_ << "' was successfully written";
- Logger_->Info(s.Str());
- }
- return true;
- } catch (const std::exception& e) {
- if (Logger_) {
- s << "Failed to write '" << Filename_ << "': " << e.what();
- Logger_->Error(s.Str());
- }
- }
-
- return false;
- }
-
- TString TDiskWriter::PrepareData(TInstant time, TStringBuf data) {
- TString toHash = WriteTimestamp(time.TimeT()) + data;
- return TDiskReader::GetHash(toHash) + toHash;
- }
-
- TString TDiskWriter::WriteTimestamp(time_t time) {
- TString res(TIMESTAMP_SIZE, 0);
- for (size_t idx = 0; idx < TIMESTAMP_SIZE; ++idx) {
- res[idx] = time & 0xFF;
- time >>= 8;
- }
- return res;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/disk_cache.h b/library/cpp/tvmauth/client/misc/disk_cache.h
deleted file mode 100644
index 9e77556f86..0000000000
--- a/library/cpp/tvmauth/client/misc/disk_cache.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-
-#include <util/datetime/base.h>
-#include <util/generic/string.h>
-
-namespace NTvmAuth {
- class ILogger;
-
- class TDiskReader {
- public:
- TDiskReader(const TString& filename, ILogger* logger = nullptr);
-
- bool Read();
-
- const TString& Data() const {
- return Data_;
- }
-
- TInstant Time() const {
- return Time_;
- }
-
- public: // for tests
- bool ParseData(TStringBuf buf);
-
- static TString GetHash(TStringBuf data);
- static time_t GetTimestamp(TStringBuf data);
-
- private:
- TString Filename_;
- ILogger* Logger_;
- TInstant Time_;
- TString Data_;
- };
-
- class TDiskWriter {
- public:
- TDiskWriter(const TString& filename, ILogger* logger = nullptr);
-
- bool Write(TStringBuf data, TInstant now = TInstant::Now());
-
- public: // for tests
- static TString PrepareData(TInstant time, TStringBuf data);
- static TString WriteTimestamp(time_t time);
-
- private:
- TString Filename_;
- ILogger* Logger_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/exponential_backoff.h b/library/cpp/tvmauth/client/misc/exponential_backoff.h
deleted file mode 100644
index 89a7a3c8ad..0000000000
--- a/library/cpp/tvmauth/client/misc/exponential_backoff.h
+++ /dev/null
@@ -1,94 +0,0 @@
-#pragma once
-
-#include <util/datetime/base.h>
-#include <util/random/normal.h>
-#include <util/system/event.h>
-
-#include <atomic>
-
-namespace NTvmAuth {
- // https://habr.com/ru/post/227225/
- class TExponentialBackoff {
- public:
- struct TSettings {
- TDuration Min;
- TDuration Max;
- double Factor = 1.001;
- double Jitter = 0;
-
- bool operator==(const TSettings& o) const {
- return Min == o.Min &&
- Max == o.Max &&
- Factor == o.Factor &&
- Jitter == o.Jitter;
- }
- };
-
- TExponentialBackoff(const TSettings& settings, bool isEnabled = true)
- : CurrentValue_(settings.Min)
- , IsEnabled_(isEnabled)
- {
- UpdateSettings(settings);
- }
-
- void UpdateSettings(const TSettings& settings) {
- Y_ENSURE(settings.Factor > 1, "factor=" << settings.Factor << ". Should be > 1");
- Y_ENSURE(settings.Jitter >= 0 && settings.Jitter < 1, "jitter should be in range [0, 1)");
-
- Min_ = settings.Min;
- Max_ = settings.Max;
- Factor_ = settings.Factor;
- Jitter_ = settings.Jitter;
- }
-
- TDuration Increase() {
- CurrentValue_ = std::min(CurrentValue_ * Factor_, Max_);
-
- double rnd = StdNormalRandom<double>();
- const bool isNegative = rnd < 0;
- rnd = std::abs(rnd);
-
- const TDuration diff = rnd * Jitter_ * CurrentValue_;
- if (isNegative) {
- CurrentValue_ -= diff;
- } else {
- CurrentValue_ += diff;
- }
-
- return CurrentValue_;
- }
-
- TDuration Decrease() {
- CurrentValue_ = std::max(CurrentValue_ / Factor_, Min_);
- return CurrentValue_;
- }
-
- void Sleep(TDuration add = TDuration()) {
- if (IsEnabled_.load(std::memory_order_relaxed)) {
- Ev_.WaitT(CurrentValue_ + add);
- }
- }
-
- void Interrupt() {
- Ev_.Signal();
- }
-
- TDuration GetCurrentValue() const {
- return CurrentValue_;
- }
-
- void SetEnabled(bool val) {
- IsEnabled_.store(val);
- }
-
- private:
- TDuration Min_;
- TDuration Max_;
- double Factor_;
- double Jitter_;
- TDuration CurrentValue_;
- std::atomic_bool IsEnabled_;
-
- TAutoEvent Ev_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/fetch_result.h b/library/cpp/tvmauth/client/misc/fetch_result.h
deleted file mode 100644
index 4b0774e92f..0000000000
--- a/library/cpp/tvmauth/client/misc/fetch_result.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#pragma once
-
-#include <library/cpp/http/simple/http_client.h>
-
-namespace NTvmAuth::NUtils {
- struct TFetchResult {
- TKeepAliveHttpClient::THttpCode Code;
- THttpHeaders Headers;
- TStringBuf Path;
- TString Response;
- TString RetrySettings;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/getter.h b/library/cpp/tvmauth/client/misc/getter.h
deleted file mode 100644
index b0327d69e9..0000000000
--- a/library/cpp/tvmauth/client/misc/getter.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#pragma once
-
-#include "checker.h"
-
-namespace NTvmAuth {
- class TServiceTicketGetter {
- public:
- TServiceTicketGetter(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- Y_ENSURE(Updater_);
- GetCache();
- }
-
- /*!
- * Fetching must enabled in TClientSettings
- * Can throw exception if cache is invalid or wrong config
- * @param dst
- */
- TString GetTicket(const TClientSettings::TAlias& dst) const {
- TServiceTicketsPtr c = GetCache();
- return GetTicketImpl(dst, c->TicketsByAlias, c->ErrorsByAlias, c->UnfetchedAliases);
- }
-
- TString GetTicket(const TTvmId dst) const {
- TServiceTicketsPtr c = GetCache();
- return GetTicketImpl(dst, c->TicketsById, c->ErrorsById, c->UnfetchedIds);
- }
-
- private:
- template <class Key, class Cont, class UnfetchedCont>
- TString GetTicketImpl(const Key& dst, const Cont& tickets, const Cont& errors, const UnfetchedCont& unfetched) const {
- auto it = tickets.find(dst);
- if (it != tickets.end()) {
- return it->second;
- }
-
- it = errors.find(dst);
- if (it != errors.end()) {
- ythrow TMissingServiceTicket()
- << "Failed to get ticket for '" << dst << "': "
- << it->second;
- }
-
- if (unfetched.contains(dst)) {
- ythrow TMissingServiceTicket()
- << "Failed to get ticket for '" << dst << "': this dst was not fetched yet.";
- }
-
- ythrow TBrokenTvmClientSettings()
- << "Destination '" << dst << "' was not specified in settings. "
- << "Check your settings (if you use Qloud/YP/tvmtool - check it's settings)";
- }
-
- private:
- TServiceTicketsPtr GetCache() const {
- TServiceTicketsPtr c = Updater_->GetCachedServiceTickets();
- Y_ENSURE_EX(c, TBrokenTvmClientSettings()
- << "Need to use TClientSettings::EnableServiceTicketsFetchOptions()");
- return c;
- }
-
- private:
- TAsyncUpdaterPtr Updater_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/last_error.cpp b/library/cpp/tvmauth/client/misc/last_error.cpp
deleted file mode 100644
index a6279bb1ef..0000000000
--- a/library/cpp/tvmauth/client/misc/last_error.cpp
+++ /dev/null
@@ -1,115 +0,0 @@
-#include "last_error.h"
-
-#include <util/string/builder.h>
-
-namespace NTvmAuth {
- TLastError::TLastError()
- : LastErrors_(MakeIntrusiveConst<TLastErrors>())
- {
- }
-
- TString TLastError::GetLastError(bool isOk, EType* type) const {
- if (isOk) {
- return OK_;
- }
-
- const TLastErrorsPtr ptr = LastErrors_.Get();
-
- for (const TLastErr& err : ptr->Errors) {
- if (err && err->first == EType::NonRetriable) {
- if (type) {
- *type = EType::NonRetriable;
- }
- return err->second;
- }
- }
-
- for (const TLastErr& err : ptr->Errors) {
- if (err) {
- if (type) {
- *type = EType::Retriable;
- }
- return err->second;
- }
- }
-
- if (type) {
- *type = EType::NonRetriable;
- }
- return "Internal client error: failed to collect last useful error message, please report this message to tvm-dev@yandex-team.ru";
- }
-
- TString TLastError::ProcessHttpError(TLastError::EScope scope,
- TStringBuf path,
- int code,
- const TString& msg) const {
- TString err = TStringBuilder() << "Path:" << path << ".Code=" << code << ": " << msg;
-
- ProcessError(code >= 400 && code < 500 ? EType::NonRetriable
- : EType::Retriable,
- scope,
- err);
-
- return err;
- }
-
- void TLastError::ProcessError(TLastError::EType type, TLastError::EScope scope, const TStringBuf msg) const {
- Update(scope, [&](TLastErr& lastError) {
- if (lastError && lastError->first == EType::NonRetriable && type == EType::Retriable) {
- return false;
- }
-
- TString err = TStringBuilder() << scope << ": " << msg;
- err.erase(std::remove(err.begin(), err.vend(), '\r'), err.vend());
- std::replace(err.begin(), err.vend(), '\n', ' ');
-
- lastError = {type, std::move(err)};
- return true;
- });
- }
-
- void TLastError::ClearError(TLastError::EScope scope) {
- Update(scope, [&](TLastErr& lastError) {
- if (!lastError) {
- return false;
- }
-
- lastError.Clear();
- return true;
- });
- }
-
- void TLastError::ClearErrors() {
- for (size_t idx = 0; idx < (size_t)EScope::COUNT; ++idx) {
- ClearError((EScope)idx);
- }
- }
-
- void TLastError::ThrowLastError() {
- EType type;
- TString err = GetLastError(false, &type);
-
- switch (type) {
- case EType::NonRetriable:
- ythrow TNonRetriableException()
- << "Failed to start TvmClient. Do not retry: "
- << err;
- case EType::Retriable:
- ythrow TRetriableException()
- << "Failed to start TvmClient. You can retry: "
- << err;
- }
- }
-
- template <typename Func>
- void TLastError::Update(TLastError::EScope scope, Func func) const {
- Y_VERIFY(scope != EScope::COUNT);
-
- TLastErrors errs = *LastErrors_.Get();
- TLastErr& lastError = errs.Errors[(size_t)scope];
-
- if (func(lastError)) {
- LastErrors_.Set(MakeIntrusiveConst<TLastErrors>(std::move(errs)));
- }
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/last_error.h b/library/cpp/tvmauth/client/misc/last_error.h
deleted file mode 100644
index b0ad33611f..0000000000
--- a/library/cpp/tvmauth/client/misc/last_error.h
+++ /dev/null
@@ -1,51 +0,0 @@
-#pragma once
-
-#include "utils.h"
-
-#include <array>
-
-namespace NTvmAuth {
- class TLastError {
- public:
- enum class EType {
- NonRetriable,
- Retriable,
- };
-
- enum class EScope {
- ServiceTickets,
- PublicKeys,
- Roles,
- TvmtoolConfig,
-
- COUNT,
- };
-
- using TLastErr = TMaybe<std::pair<EType, TString>>;
-
- struct TLastErrors: public TAtomicRefCount<TLastErrors> {
- std::array<TLastErr, (int)EScope::COUNT> Errors;
- };
- using TLastErrorsPtr = TIntrusiveConstPtr<TLastErrors>;
-
- public:
- TLastError();
-
- TString GetLastError(bool isOk, EType* type = nullptr) const;
-
- TString ProcessHttpError(EScope scope, TStringBuf path, int code, const TString& msg) const;
- void ProcessError(EType type, EScope scope, const TStringBuf msg) const;
- void ClearError(EScope scope);
- void ClearErrors();
- void ThrowLastError();
-
- private:
- template <typename Func>
- void Update(EScope scope, Func func) const;
-
- private:
- const TString OK_ = "OK";
-
- mutable NUtils::TProtectedValue<TLastErrorsPtr> LastErrors_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/proc_info.cpp b/library/cpp/tvmauth/client/misc/proc_info.cpp
deleted file mode 100644
index e2e5ec15b9..0000000000
--- a/library/cpp/tvmauth/client/misc/proc_info.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-#include "proc_info.h"
-
-#include <library/cpp/tvmauth/version.h>
-
-#include <library/cpp/string_utils/quote/quote.h>
-
-#include <util/stream/file.h>
-#include <util/string/cast.h>
-#include <util/system/getpid.h>
-
-namespace NTvmAuth::NUtils {
- void TProcInfo::AddToRequest(IOutputStream& out) const {
- out << "_pid=" << Pid;
- if (ProcessName) {
- out << "&_procces_name=" << *ProcessName;
- }
- out << "&lib_version=client_" << VersionPrefix << LibVersion();
- }
-
- TProcInfo TProcInfo::Create(const TString& versionPrefix) {
- TProcInfo res;
- res.Pid = IntToString<10>(GetPID());
- res.ProcessName = GetProcessName();
- res.VersionPrefix = versionPrefix;
- return res;
- }
-
- std::optional<TString> TProcInfo::GetProcessName() {
- try {
- // works only for linux
- TFileInput proc("/proc/self/status");
-
- TString line;
- while (proc.ReadLine(line)) {
- TStringBuf buf(line);
- if (!buf.SkipPrefix("Name:")) {
- continue;
- }
-
- while (buf && isspace(buf.front())) {
- buf.Skip(1);
- }
-
- TString res(buf);
- CGIEscape(res);
- return res;
- }
- } catch (...) {
- }
-
- return {};
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/proc_info.h b/library/cpp/tvmauth/client/misc/proc_info.h
deleted file mode 100644
index b1526e5c47..0000000000
--- a/library/cpp/tvmauth/client/misc/proc_info.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#pragma once
-
-#include <util/generic/string.h>
-
-#include <optional>
-
-namespace NTvmAuth::NUtils {
- struct TProcInfo {
- TString Pid;
- std::optional<TString> ProcessName;
- TString VersionPrefix;
-
- void AddToRequest(IOutputStream& out) const;
-
- static TProcInfo Create(const TString& versionPrefix);
- static std::optional<TString> GetProcessName();
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/retry_settings/v1/CMakeLists.txt b/library/cpp/tvmauth/client/misc/retry_settings/v1/CMakeLists.txt
deleted file mode 100644
index 2248564f12..0000000000
--- a/library/cpp/tvmauth/client/misc/retry_settings/v1/CMakeLists.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(misc-retry_settings-v1)
-target_link_libraries(misc-retry_settings-v1 PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-protobuf
-)
-target_proto_messages(misc-retry_settings-v1 PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/client/misc/retry_settings/v1/settings.proto
-)
-target_proto_addincls(misc-retry_settings-v1
- ./
- ${CMAKE_SOURCE_DIR}/
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(misc-retry_settings-v1
- --cpp_out=${CMAKE_BINARY_DIR}/
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
-)
diff --git a/library/cpp/tvmauth/client/misc/retry_settings/v1/settings.proto b/library/cpp/tvmauth/client/misc/retry_settings/v1/settings.proto
deleted file mode 100644
index 72817847a6..0000000000
--- a/library/cpp/tvmauth/client/misc/retry_settings/v1/settings.proto
+++ /dev/null
@@ -1,21 +0,0 @@
-syntax = "proto2";
-
-package retry_settings.v1;
-
-option cc_enable_arenas = true;
-option go_package = "a.yandex-team.ru/library/cpp/tvmauth/client/misc/retry_settings/v1";
-
-message Settings {
- optional uint32 exponential_backoff_min_sec = 1;
- optional uint32 exponential_backoff_max_sec = 2;
- optional double exponential_backoff_factor = 3;
- optional double exponential_backoff_jitter = 4;
- optional uint32 max_random_sleep_default = 5;
- optional uint32 max_random_sleep_when_ok = 12;
- optional uint32 retries_on_start = 6;
- optional uint32 worker_awaking_period_sec = 7;
- optional uint32 dsts_limit = 8;
- optional uint32 retries_in_background = 9;
- optional uint32 roles_update_period_sec = 10;
- optional uint32 roles_warn_period_sec = 11;
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/decoder.cpp b/library/cpp/tvmauth/client/misc/roles/decoder.cpp
deleted file mode 100644
index 6337fb91c2..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/decoder.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-#include "decoder.h"
-
-#include <library/cpp/tvmauth/client/misc/utils.h>
-
-#include <library/cpp/openssl/crypto/sha.h>
-#include <library/cpp/streams/brotli/brotli.h>
-#include <library/cpp/streams/zstd/zstd.h>
-
-#include <util/generic/yexception.h>
-#include <util/stream/zlib.h>
-#include <util/string/ascii.h>
-
-namespace NTvmAuth::NRoles {
- TString TDecoder::Decode(const TStringBuf codec, TString&& blob) {
- if (codec.empty()) {
- return std::move(blob);
- }
-
- const TCodecInfo info = ParseCodec(codec);
- TString decoded = DecodeImpl(info.Type, blob);
-
- VerifySize(decoded, info.Size);
- VerifyChecksum(decoded, info.Sha256);
-
- return decoded;
- }
-
- TDecoder::TCodecInfo TDecoder::ParseCodec(TStringBuf codec) {
- const char delim = ':';
-
- const TStringBuf version = codec.NextTok(delim);
- Y_ENSURE(version == "1",
- "unknown codec format version; known: 1; got: " << version);
-
- TCodecInfo res;
- res.Type = codec.NextTok(delim);
- Y_ENSURE(res.Type, "codec type is empty");
-
- const TStringBuf size = codec.NextTok(delim);
- Y_ENSURE(TryIntFromString<10>(size, res.Size),
- "decoded blob size is not number");
-
- res.Sha256 = codec;
- const size_t expectedSha256Size = 2 * NOpenSsl::NSha256::DIGEST_LENGTH;
- Y_ENSURE(res.Sha256.size() == expectedSha256Size,
- "sha256 of decoded blob has invalid length: expected "
- << expectedSha256Size << ", got " << res.Sha256.size());
-
- return res;
- }
-
- TString TDecoder::DecodeImpl(TStringBuf codec, const TString& blob) {
- if (AsciiEqualsIgnoreCase(codec, "brotli")) {
- return DecodeBrolti(blob);
- } else if (AsciiEqualsIgnoreCase(codec, "gzip")) {
- return DecodeGzip(blob);
- } else if (AsciiEqualsIgnoreCase(codec, "zstd")) {
- return DecodeZstd(blob);
- }
-
- ythrow yexception() << "unknown codec: '" << codec << "'";
- }
-
- TString TDecoder::DecodeBrolti(const TString& blob) {
- TStringInput in(blob);
- return TBrotliDecompress(&in).ReadAll();
- }
-
- TString TDecoder::DecodeGzip(const TString& blob) {
- TStringInput in(blob);
- return TZLibDecompress(&in).ReadAll();
- }
-
- TString TDecoder::DecodeZstd(const TString& blob) {
- TStringInput in(blob);
- return TZstdDecompress(&in).ReadAll();
- }
-
- void TDecoder::VerifySize(const TStringBuf decoded, size_t expected) {
- Y_ENSURE(expected == decoded.size(),
- "Decoded blob has bad size: expected " << expected << ", actual " << decoded.size());
- }
-
- void TDecoder::VerifyChecksum(const TStringBuf decoded, const TStringBuf expected) {
- using namespace NOpenSsl::NSha256;
-
- const TDigest dig = Calc(decoded);
- const TString actual = NUtils::ToHex(TStringBuf((char*)dig.data(), dig.size()));
-
- Y_ENSURE(AsciiEqualsIgnoreCase(actual, expected),
- "Decoded blob has bad sha256: expected=" << expected << ", actual=" << actual);
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/decoder.h b/library/cpp/tvmauth/client/misc/roles/decoder.h
deleted file mode 100644
index de5cdb37e0..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/decoder.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#pragma once
-
-#include <util/generic/string.h>
-
-namespace NTvmAuth::NRoles {
- class TDecoder {
- public:
- static TString Decode(const TStringBuf codec, TString&& blob);
-
- public:
- struct TCodecInfo {
- TStringBuf Type;
- size_t Size = 0;
- TStringBuf Sha256;
-
- bool operator==(const TCodecInfo& o) const {
- return Type == o.Type &&
- Size == o.Size &&
- Sha256 == o.Sha256;
- }
- };
-
- static TCodecInfo ParseCodec(TStringBuf codec);
- static TString DecodeImpl(TStringBuf codec, const TString& blob);
- static TString DecodeBrolti(const TString& blob);
- static TString DecodeGzip(const TString& blob);
- static TString DecodeZstd(const TString& blob);
-
- static void VerifySize(const TStringBuf decoded, size_t expected);
- static void VerifyChecksum(const TStringBuf decoded, const TStringBuf expected);
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/entities_index.cpp b/library/cpp/tvmauth/client/misc/roles/entities_index.cpp
deleted file mode 100644
index c9b72c3a17..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/entities_index.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-#include "entities_index.h"
-
-#include <util/stream/str.h>
-
-#include <set>
-
-namespace NTvmAuth::NRoles {
- TEntitiesIndex::TStage::TStage(const std::set<TString>& k)
- : Keys_(k.begin(), k.end())
- {
- }
-
- // TODO TStringBuf
- bool TEntitiesIndex::TStage::GetNextKeySet(std::vector<TString>& out) {
- out.clear();
- out.reserve(Keys_.size());
-
- ++Id_;
- for (size_t idx = 0; idx < Keys_.size(); ++idx) {
- bool need = (Id_ >> idx) & 0x01;
-
- if (need) {
- out.push_back(Keys_[idx]);
- }
- }
-
- return !out.empty();
- }
-
- TEntitiesIndex::TEntitiesIndex(const std::vector<TEntityPtr>& entities) {
- const std::set<TString> uniqueKeys = GetUniqueSortedKeys(entities);
- Idx_.Entities = entities;
- Idx_.SubTree.reserve(uniqueKeys.size() * entities.size());
-
- TStage stage(uniqueKeys);
- std::vector<TString> keyset;
- while (stage.GetNextKeySet(keyset)) {
- for (const TEntityPtr& e : entities) {
- TSubTree* currentBranch = &Idx_;
-
- for (const TString& key : keyset) {
- auto it = e->find(key);
- if (it == e->end()) {
- continue;
- }
-
- auto [i, ok] = currentBranch->SubTree.emplace(
- TKeyValue{it->first, it->second},
- TSubTree());
-
- currentBranch = &i->second;
- currentBranch->Entities.push_back(e);
- }
- }
- }
-
- MakeUnique(Idx_);
- }
-
- std::set<TString> TEntitiesIndex::GetUniqueSortedKeys(const std::vector<TEntityPtr>& entities) {
- std::set<TString> res;
-
- for (const TEntityPtr& e : entities) {
- for (const auto& [key, value] : *e) {
- res.insert(key);
- }
- }
-
- return res;
- }
-
- void TEntitiesIndex::MakeUnique(TSubTree& branch) {
- auto& vec = branch.Entities;
- std::sort(vec.begin(), vec.end());
- vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
-
- for (auto& [_, restPart] : branch.SubTree) {
- MakeUnique(restPart);
- }
- }
-
- static void Print(const TEntitiesIndex::TSubTree& part, IOutputStream& out, size_t offset = 0) {
- std::vector<std::pair<TKeyValue, const TEntitiesIndex::TSubTree*>> vec;
- vec.reserve(part.SubTree.size());
-
- for (const auto& [key, value] : part.SubTree) {
- vec.push_back({key, &value});
- }
-
- std::sort(vec.begin(), vec.end(), [](const auto& l, const auto& r) {
- if (l.first.Key < r.first.Key) {
- return true;
- }
- if (l.first.Value < r.first.Value) {
- return true;
- }
- return false;
- });
-
- for (const auto& [key, value] : vec) {
- out << TString(offset, ' ') << "\"" << key.Key << "/" << key.Value << "\"" << Endl;
- Print(*value, out, offset + 4);
- }
- }
-
- TString TEntitiesIndex::PrintDebugString() const {
- TStringStream res;
- res << Endl;
-
- Print(Idx_, res);
-
- return res.Str();
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/entities_index.h b/library/cpp/tvmauth/client/misc/roles/entities_index.h
deleted file mode 100644
index bf42750d52..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/entities_index.h
+++ /dev/null
@@ -1,107 +0,0 @@
-#pragma once
-
-#include "types.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <set>
-#include <vector>
-
-namespace NTvmAuth::NRoles {
- class TEntitiesIndex: TMoveOnly {
- public:
- struct TSubTree;
- using TIdxByAttrs = THashMap<TKeyValue, TSubTree>;
-
- struct TSubTree {
- std::vector<TEntityPtr> Entities;
- TIdxByAttrs SubTree;
- };
-
- class TStage {
- public:
- TStage(const std::set<TString>& k);
-
- bool GetNextKeySet(std::vector<TString>& out);
-
- private:
- std::vector<TString> Keys_;
- size_t Id_ = 0;
- };
-
- public:
- TEntitiesIndex(const std::vector<TEntityPtr>& entities);
-
- /**
- * Iterators must be to sorted unique key/value
- */
- template <typename Iterator>
- bool ContainsExactEntity(Iterator begin, Iterator end) const;
-
- /**
- * Iterators must be to sorted unique key/value
- */
- template <typename Iterator>
- const std::vector<TEntityPtr>& GetEntitiesWithAttrs(Iterator begin, Iterator end) const;
-
- public: // for tests
- static std::set<TString> GetUniqueSortedKeys(const std::vector<TEntityPtr>& entities);
- static void MakeUnique(TEntitiesIndex::TSubTree& branch);
-
- TString PrintDebugString() const;
-
- private:
- template <typename Iterator>
- const TSubTree* FindSubtree(Iterator begin, Iterator end, size_t& size) const;
-
- private:
- TSubTree Idx_;
- std::vector<TEntityPtr> EmptyResult_;
- };
-
- template <typename Iterator>
- bool TEntitiesIndex::ContainsExactEntity(Iterator begin, Iterator end) const {
- size_t size = 0;
- const TSubTree* subtree = FindSubtree(begin, end, size);
- if (!subtree) {
- return false;
- }
-
- auto res = std::find_if(
- subtree->Entities.begin(),
- subtree->Entities.end(),
- [size](const auto& e) { return size == e->size(); });
- return res != subtree->Entities.end();
- }
-
- template <typename Iterator>
- const std::vector<TEntityPtr>& TEntitiesIndex::GetEntitiesWithAttrs(Iterator begin, Iterator end) const {
- size_t size = 0;
- const TSubTree* subtree = FindSubtree(begin, end, size);
- if (!subtree) {
- return EmptyResult_;
- }
-
- return subtree->Entities;
- }
-
- template <typename Iterator>
- const TEntitiesIndex::TSubTree* TEntitiesIndex::FindSubtree(Iterator begin,
- Iterator end,
- size_t& size) const {
- const TSubTree* subtree = &Idx_;
- size = 0;
-
- for (auto attr = begin; attr != end; ++attr) {
- auto it = subtree->SubTree.find(TKeyValueView{attr->first, attr->second});
- if (it == subtree->SubTree.end()) {
- return nullptr;
- }
-
- ++size;
- subtree = &it->second;
- }
-
- return subtree;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/parser.cpp b/library/cpp/tvmauth/client/misc/roles/parser.cpp
deleted file mode 100644
index 28faf4c057..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/parser.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-#include "parser.h"
-
-#include <library/cpp/json/json_reader.h>
-
-#include <util/string/cast.h>
-
-namespace NTvmAuth::NRoles {
- static void GetRequiredValue(const NJson::TJsonValue& doc,
- TStringBuf key,
- NJson::TJsonValue& obj) {
- Y_ENSURE(doc.GetValue(key, &obj), "Missing '" << key << "'");
- }
-
- static ui64 GetRequiredUInt(const NJson::TJsonValue& doc,
- TStringBuf key) {
- NJson::TJsonValue obj;
- GetRequiredValue(doc, key, obj);
- Y_ENSURE(obj.IsUInteger(), "key '" << key << "' must be uint");
- return obj.GetUInteger();
- }
-
- static bool GetOptionalMap(const NJson::TJsonValue& doc,
- TStringBuf key,
- NJson::TJsonValue& obj) {
- if (!doc.GetValue(key, &obj)) {
- return false;
- }
-
- Y_ENSURE(obj.IsMap(), "'" << key << "' must be object");
- return true;
- }
-
- TRolesPtr TParser::Parse(TRawPtr decodedBlob) {
- try {
- return ParseImpl(decodedBlob);
- } catch (const std::exception& e) {
- throw yexception() << "Failed to parse roles from tirole: " << e.what()
- << ". '" << *decodedBlob << "'";
- }
- }
-
- TRolesPtr TParser::ParseImpl(TRawPtr decodedBlob) {
- NJson::TJsonValue doc;
- Y_ENSURE(NJson::ReadJsonTree(*decodedBlob, &doc), "Invalid json");
- Y_ENSURE(doc.IsMap(), "Json must be object");
-
- TRoles::TTvmConsumers tvm = GetConsumers<TTvmId>(doc, "tvm");
- TRoles::TUserConsumers user = GetConsumers<TUid>(doc, "user");
-
- // fetch it last to provide more correct apply instant
- TRoles::TMeta meta = GetMeta(doc);
-
- return std::make_shared<TRoles>(
- std::move(meta),
- std::move(tvm),
- std::move(user),
- std::move(decodedBlob));
- }
-
- TRoles::TMeta TParser::GetMeta(const NJson::TJsonValue& doc) {
- TRoles::TMeta res;
-
- NJson::TJsonValue obj;
- GetRequiredValue(doc, "revision", obj);
- if (obj.IsString()) {
- res.Revision = obj.GetString();
- } else if (obj.IsUInteger()) {
- res.Revision = ToString(obj.GetUInteger());
- } else {
- ythrow yexception() << "'revision' has unexpected type: " << obj.GetType();
- }
-
- res.BornTime = TInstant::Seconds(GetRequiredUInt(doc, "born_date"));
-
- return res;
- }
-
- template <typename Id>
- THashMap<Id, TConsumerRolesPtr> TParser::GetConsumers(const NJson::TJsonValue& doc,
- TStringBuf type) {
- THashMap<Id, TConsumerRolesPtr> res;
-
- NJson::TJsonValue obj;
- if (!GetOptionalMap(doc, type, obj)) {
- return res;
- }
-
- for (const auto& [key, value] : obj.GetMap()) {
- Y_ENSURE(value.IsMap(),
- "roles for consumer must be map: '" << key << "' is " << value.GetType());
-
- Id id = 0;
- Y_ENSURE(TryIntFromString<10>(key, id),
- "id must be valid positive number of proper size for "
- << type << ". got '"
- << key << "'");
-
- Y_ENSURE(res.emplace(id, GetConsumer(value, key)).second,
- "consumer duplicate detected: '" << key << "' for " << type);
- }
-
- return res;
- }
-
- TConsumerRolesPtr TParser::GetConsumer(const NJson::TJsonValue& obj, TStringBuf consumer) {
- TEntitiesByRoles entities;
-
- for (const auto& [key, value] : obj.GetMap()) {
- Y_ENSURE(value.IsArray(),
- "entities for roles must be array: '" << key << "' is " << value.GetType());
-
- entities.emplace(key, GetEntities(value, consumer, key));
- }
-
- return std::make_shared<TConsumerRoles>(std::move(entities));
- }
-
- TEntitiesPtr TParser::GetEntities(const NJson::TJsonValue& obj,
- TStringBuf consumer,
- TStringBuf role) {
- std::vector<TEntityPtr> entities;
- entities.reserve(obj.GetArray().size());
-
- for (const NJson::TJsonValue& e : obj.GetArray()) {
- Y_ENSURE(e.IsMap(),
- "role entity for role must be map: consumer '"
- << consumer << "' with role '" << role << "' has " << e.GetType());
-
- entities.push_back(GetEntity(e, consumer, role));
- }
-
- return std::make_shared<TEntities>(TEntities(entities));
- }
-
- TEntityPtr TParser::GetEntity(const NJson::TJsonValue& obj, TStringBuf consumer, TStringBuf role) {
- TEntityPtr res = std::make_shared<TEntity>();
-
- for (const auto& [key, value] : obj.GetMap()) {
- Y_ENSURE(value.IsString(),
- "entity is map (str->str), got value "
- << value.GetType() << ". consumer '"
- << consumer << "' with role '" << role << "'");
-
- res->emplace(key, value.GetString());
- }
-
- return res;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/parser.h b/library/cpp/tvmauth/client/misc/roles/parser.h
deleted file mode 100644
index 0982ba78c6..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/parser.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#pragma once
-
-#include "roles.h"
-#include "types.h"
-
-namespace NJson {
- class TJsonValue;
-}
-
-namespace NTvmAuth::NRoles {
- class TParser {
- public:
- static TRolesPtr Parse(TRawPtr decodedBlob);
-
- public:
- static TRolesPtr ParseImpl(TRawPtr decodedBlob);
- static TRoles::TMeta GetMeta(const NJson::TJsonValue& doc);
-
- template <typename Id>
- static THashMap<Id, TConsumerRolesPtr> GetConsumers(
- const NJson::TJsonValue& doc,
- TStringBuf key);
-
- static TConsumerRolesPtr GetConsumer(
- const NJson::TJsonValue& obj,
- TStringBuf consumer);
- static TEntitiesPtr GetEntities(
- const NJson::TJsonValue& obj,
- TStringBuf consumer,
- TStringBuf role);
- static TEntityPtr GetEntity(
- const NJson::TJsonValue& obj,
- TStringBuf consumer,
- TStringBuf role);
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/roles.cpp b/library/cpp/tvmauth/client/misc/roles/roles.cpp
deleted file mode 100644
index 0761033104..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/roles.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-#include "roles.h"
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-namespace NTvmAuth::NRoles {
- TRoles::TRoles(TMeta&& meta,
- TTvmConsumers tvm,
- TUserConsumers user,
- TRawPtr raw)
- : Meta_(std::move(meta))
- , TvmIds_(std::move(tvm))
- , Users_(std::move(user))
- , Raw_(std::move(raw))
- {
- Y_ENSURE(Raw_);
- }
-
- TConsumerRolesPtr TRoles::GetRolesForService(const TCheckedServiceTicket& t) const {
- Y_ENSURE_EX(t,
- TIllegalUsage() << "Service ticket must be valid, got: " << t.GetStatus());
- auto it = TvmIds_.find(t.GetSrc());
- return it == TvmIds_.end() ? TConsumerRolesPtr() : it->second;
- }
-
- TConsumerRolesPtr TRoles::GetRolesForUser(const TCheckedUserTicket& t,
- std::optional<TUid> selectedUid) const {
- Y_ENSURE_EX(t,
- TIllegalUsage() << "User ticket must be valid, got: " << t.GetStatus());
- Y_ENSURE_EX(t.GetEnv() == EBlackboxEnv::ProdYateam,
- TIllegalUsage() << "User ticket must be from ProdYateam, got from " << t.GetEnv());
-
- TUid uid = t.GetDefaultUid();
- if (selectedUid) {
- auto it = std::find(t.GetUids().begin(), t.GetUids().end(), *selectedUid);
- Y_ENSURE_EX(it != t.GetUids().end(),
- TIllegalUsage() << "selectedUid must be in user ticket but it's not: "
- << *selectedUid);
- uid = *selectedUid;
- }
-
- auto it = Users_.find(uid);
- return it == Users_.end() ? TConsumerRolesPtr() : it->second;
- }
-
- const TRoles::TMeta& TRoles::GetMeta() const {
- return Meta_;
- }
-
- const TString& TRoles::GetRaw() const {
- return *Raw_;
- }
-
- bool TRoles::CheckServiceRole(const TCheckedServiceTicket& t,
- const TStringBuf roleName) const {
- TConsumerRolesPtr c = GetRolesForService(t);
- return c ? c->HasRole(roleName) : false;
- }
-
- bool TRoles::CheckUserRole(const TCheckedUserTicket& t,
- const TStringBuf roleName,
- std::optional<TUid> selectedUid) const {
- TConsumerRolesPtr c = GetRolesForUser(t, selectedUid);
- return c ? c->HasRole(roleName) : false;
- }
-
- bool TRoles::CheckServiceRoleForExactEntity(const TCheckedServiceTicket& t,
- const TStringBuf roleName,
- const TEntity& exactEntity) const {
- TConsumerRolesPtr c = GetRolesForService(t);
- return c ? c->CheckRoleForExactEntity(roleName, exactEntity) : false;
- }
-
- bool TRoles::CheckUserRoleForExactEntity(const TCheckedUserTicket& t,
- const TStringBuf roleName,
- const TEntity& exactEntity,
- std::optional<TUid> selectedUid) const {
- TConsumerRolesPtr c = GetRolesForUser(t, selectedUid);
- return c ? c->CheckRoleForExactEntity(roleName, exactEntity) : false;
- }
-
- TConsumerRoles::TConsumerRoles(TEntitiesByRoles roles)
- : Roles_(std::move(roles))
- {
- }
-
- bool TConsumerRoles::CheckRoleForExactEntity(const TStringBuf roleName,
- const TEntity& exactEntity) const {
- auto it = Roles_.find(roleName);
- if (it == Roles_.end()) {
- return false;
- }
-
- return it->second->Contains(exactEntity);
- }
-
- TEntities::TEntities(TEntitiesIndex idx)
- : Idx_(std::move(idx))
- {
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/roles.h b/library/cpp/tvmauth/client/misc/roles/roles.h
deleted file mode 100644
index 6d510ee8a1..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/roles.h
+++ /dev/null
@@ -1,186 +0,0 @@
-#pragma once
-
-#include "entities_index.h"
-#include "types.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/type.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/array_ref.h>
-#include <util/generic/hash.h>
-
-#include <vector>
-
-namespace NTvmAuth {
- class TCheckedServiceTicket;
- class TCheckedUserTicket;
-}
-
-namespace NTvmAuth::NRoles {
- class TRoles {
- public:
- struct TMeta {
- TString Revision;
- TInstant BornTime;
- TInstant Applied = TInstant::Now();
- };
-
- using TTvmConsumers = THashMap<TTvmId, TConsumerRolesPtr>;
- using TUserConsumers = THashMap<TUid, TConsumerRolesPtr>;
-
- TRoles(TMeta&& meta,
- TTvmConsumers tvm,
- TUserConsumers user,
- TRawPtr raw);
-
- /**
- * @return ptr to roles. It will be nullptr if there are no roles
- */
- TConsumerRolesPtr GetRolesForService(const TCheckedServiceTicket& t) const;
-
- /**
- * @return ptr to roles. It will be nullptr if there are no roles
- */
- TConsumerRolesPtr GetRolesForUser(const TCheckedUserTicket& t,
- std::optional<TUid> selectedUid = {}) const;
-
- const TMeta& GetMeta() const;
- const TString& GetRaw() const;
-
- public: // shortcuts
- /**
- * @brief CheckServiceRole() is shortcut for simple role checking - for any possible entity
- */
- bool CheckServiceRole(
- const TCheckedServiceTicket& t,
- const TStringBuf roleName) const;
-
- /**
- * @brief CheckUserRole() is shortcut for simple role checking - for any possible entity
- */
- bool CheckUserRole(
- const TCheckedUserTicket& t,
- const TStringBuf roleName,
- std::optional<TUid> selectedUid = {}) const;
-
- /**
- * @brief CheckServiceRoleForExactEntity() is shortcut for simple role checking for exact entity
- */
- bool CheckServiceRoleForExactEntity(
- const TCheckedServiceTicket& t,
- const TStringBuf roleName,
- const TEntity& exactEntity) const;
-
- /**
- * @brief CheckUserRoleForExactEntity() is shortcut for simple role checking for exact entity
- */
- bool CheckUserRoleForExactEntity(
- const TCheckedUserTicket& t,
- const TStringBuf roleName,
- const TEntity& exactEntity,
- std::optional<TUid> selectedUid = {}) const;
-
- private:
- TMeta Meta_;
- TTvmConsumers TvmIds_;
- TUserConsumers Users_;
- TRawPtr Raw_;
- };
-
- class TConsumerRoles {
- public:
- TConsumerRoles(TEntitiesByRoles roles);
-
- bool HasRole(const TStringBuf roleName) const {
- return Roles_.contains(roleName);
- }
-
- const TEntitiesByRoles& GetRoles() const {
- return Roles_;
- }
-
- /**
- * @return ptr to entries. It will be nullptr if there is no role
- */
- TEntitiesPtr GetEntitiesForRole(const TStringBuf roleName) const {
- auto it = Roles_.find(roleName);
- return it == Roles_.end() ? TEntitiesPtr() : it->second;
- }
-
- /**
- * @brief CheckRoleForExactEntity() is shortcut for simple role checking for exact entity
- */
- bool CheckRoleForExactEntity(const TStringBuf roleName,
- const TEntity& exactEntity) const;
-
- private:
- TEntitiesByRoles Roles_;
- };
-
- class TEntities {
- public:
- TEntities(TEntitiesIndex idx);
-
- /**
- * @brief Contains() provides info about entity presence
- */
- bool Contains(const TEntity& exactEntity) const {
- return Idx_.ContainsExactEntity(exactEntity.begin(), exactEntity.end());
- }
-
- /**
- * @brief The same as Contains()
- * It checks span for sorted and unique properties.
- */
- template <class StrKey = TString, class StrValue = TString>
- bool ContainsSortedUnique(
- const TArrayRef<const std::pair<StrKey, StrValue>>& exactEntity) const {
- CheckSpan(exactEntity);
- return Idx_.ContainsExactEntity(exactEntity.begin(), exactEntity.end());
- }
-
- /**
- * @brief GetEntitiesWithAttrs() collects entities with ALL attributes from `attrs`
- */
- template <class StrKey = TString, class StrValue = TString>
- const std::vector<TEntityPtr>& GetEntitiesWithAttrs(
- const std::map<StrKey, StrValue>& attrs) const {
- return Idx_.GetEntitiesWithAttrs(attrs.begin(), attrs.end());
- }
-
- /**
- * @brief The same as GetEntitiesWithAttrs()
- * It checks span for sorted and unique properties.
- */
- template <class StrKey = TString, class StrValue = TString>
- const std::vector<TEntityPtr>& GetEntitiesWithSortedUniqueAttrs(
- const TArrayRef<const std::pair<StrKey, StrValue>>& attrs) const {
- CheckSpan(attrs);
- return Idx_.GetEntitiesWithAttrs(attrs.begin(), attrs.end());
- }
-
- private:
- template <class StrKey, class StrValue>
- static void CheckSpan(const TArrayRef<const std::pair<StrKey, StrValue>>& attrs) {
- if (attrs.empty()) {
- return;
- }
-
- auto prev = attrs.begin();
- for (auto it = prev + 1; it != attrs.end(); ++it) {
- Y_ENSURE_EX(prev->first != it->first,
- TIllegalUsage() << "attrs are not unique: '" << it->first << "'");
- Y_ENSURE_EX(prev->first < it->first,
- TIllegalUsage() << "attrs are not sorted: '" << prev->first
- << "' before '" << it->first << "'");
-
- prev = it;
- }
- }
-
- private:
- TEntitiesIndex Idx_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/roles/types.h b/library/cpp/tvmauth/client/misc/roles/types.h
deleted file mode 100644
index de0745e72e..0000000000
--- a/library/cpp/tvmauth/client/misc/roles/types.h
+++ /dev/null
@@ -1,70 +0,0 @@
-#pragma once
-
-#include <util/generic/hash_set.h>
-
-#include <map>
-#include <memory>
-
-namespace NTvmAuth::NRoles {
- using TEntity = std::map<TString, TString>;
- using TEntityPtr = std::shared_ptr<TEntity>;
-
- class TEntities;
- using TEntitiesPtr = std::shared_ptr<TEntities>;
-
- using TEntitiesByRoles = THashMap<TString, TEntitiesPtr>;
-
- class TConsumerRoles;
- using TConsumerRolesPtr = std::shared_ptr<TConsumerRoles>;
-
- class TRoles;
- using TRolesPtr = std::shared_ptr<TRoles>;
-
- using TRawPtr = std::shared_ptr<TString>;
-
- template <class T>
- struct TKeyValueBase {
- T Key;
- T Value;
-
- template <typename U>
- bool operator==(const TKeyValueBase<U>& o) const {
- return Key == o.Key && Value == o.Value;
- }
- };
-
- using TKeyValue = TKeyValueBase<TString>;
- using TKeyValueView = TKeyValueBase<TStringBuf>;
-}
-
-// Traits
-
-template <>
-struct THash<NTvmAuth::NRoles::TKeyValue> {
- std::size_t operator()(const NTvmAuth::NRoles::TKeyValue& e) const {
- return std::hash<std::string_view>()(e.Key) + std::hash<std::string_view>()(e.Value);
- }
-
- std::size_t operator()(const NTvmAuth::NRoles::TKeyValueView& e) const {
- return std::hash<std::string_view>()(e.Key) + std::hash<std::string_view>()(e.Value);
- }
-};
-
-template <>
-struct TEqualTo<NTvmAuth::NRoles::TKeyValue> {
- using is_transparent = std::true_type;
-
- template <typename T, typename U>
- bool operator()(const NTvmAuth::NRoles::TKeyValueBase<T>& l,
- const NTvmAuth::NRoles::TKeyValueBase<U>& r) {
- return l == r;
- }
-};
-
-inline bool operator<(const NTvmAuth::NRoles::TEntityPtr& l, const NTvmAuth::NRoles::TEntityPtr& r) {
- return *l < *r;
-}
-
-inline bool operator==(const NTvmAuth::NRoles::TEntityPtr& l, const NTvmAuth::NRoles::TEntityPtr& r) {
- return *l == *r;
-}
diff --git a/library/cpp/tvmauth/client/misc/settings.h b/library/cpp/tvmauth/client/misc/settings.h
deleted file mode 100644
index 8fae6c34d3..0000000000
--- a/library/cpp/tvmauth/client/misc/settings.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#pragma once
-
-#include <util/generic/fwd.h>
-
-namespace NTvmAuth {
- class TClientSettings {
- public:
- /*!
- * Look at description in relevant settings: NTvmApi::TClientSettings or NTvmTool::TClientSettings
- */
- using TAlias = TString;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/src_checker.h b/library/cpp/tvmauth/client/misc/src_checker.h
deleted file mode 100644
index 25e8e72602..0000000000
--- a/library/cpp/tvmauth/client/misc/src_checker.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#pragma once
-
-#include "async_updater.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/src/service_impl.h>
-
-namespace NTvmAuth {
- class TSrcChecker {
- public:
- TSrcChecker(TAsyncUpdaterPtr updater)
- : Updater_(std::move(updater))
- {
- Y_ENSURE(Updater_);
- GetCache();
- }
-
- /*!
- * Checking must be enabled in TClientSettings
- * Can throw exception if cache is out of date or wrong config
- * @param ticket
- */
- TCheckedServiceTicket Check(TCheckedServiceTicket ticket) const {
- NRoles::TConsumerRolesPtr roles = GetCache()->GetRolesForService(ticket);
- if (roles) {
- return ticket;
- }
-
- TServiceTicketImplPtr impl = THolder(NInternal::TCanningKnife::GetS(ticket));
- impl->SetStatus(ETicketStatus::NoRoles);
- return TCheckedServiceTicket(std::move(impl));
- }
-
- private:
- NRoles::TRolesPtr GetCache() const {
- NRoles::TRolesPtr c = Updater_->GetRoles();
- Y_ENSURE_EX(c, TBrokenTvmClientSettings() << "Need to use TClientSettings::EnableRolesFetching()");
- return c;
- }
-
- private:
- TAsyncUpdaterPtr Updater_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/threaded_updater.cpp b/library/cpp/tvmauth/client/misc/threaded_updater.cpp
deleted file mode 100644
index 5d21ce67a7..0000000000
--- a/library/cpp/tvmauth/client/misc/threaded_updater.cpp
+++ /dev/null
@@ -1,111 +0,0 @@
-#include "threaded_updater.h"
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <util/string/builder.h>
-#include <util/system/spin_wait.h>
-#include <util/system/thread.h>
-
-namespace NTvmAuth {
- TThreadedUpdaterBase::TThreadedUpdaterBase(TDuration workerAwakingPeriod,
- TLoggerPtr logger,
- const TString& url,
- ui16 port,
- TDuration socketTimeout,
- TDuration connectTimeout)
- : WorkerAwakingPeriod_(workerAwakingPeriod)
- , Logger_(std::move(logger))
- , TvmUrl_(url)
- , TvmPort_(port)
- , TvmSocketTimeout_(socketTimeout)
- , TvmConnectTimeout_(connectTimeout)
- , IsStopped_(true)
- {
- Y_ENSURE_EX(Logger_, TNonRetriableException() << "Logger is required");
-
- ServiceTicketsDurations_.RefreshPeriod = TDuration::Hours(1);
- ServiceTicketsDurations_.Expiring = TDuration::Hours(2);
- ServiceTicketsDurations_.Invalid = TDuration::Hours(11);
-
- PublicKeysDurations_.RefreshPeriod = TDuration::Days(1);
- PublicKeysDurations_.Expiring = TDuration::Days(2);
- PublicKeysDurations_.Invalid = TDuration::Days(6);
- }
-
- TThreadedUpdaterBase::~TThreadedUpdaterBase() {
- StopWorker();
- }
-
- void TThreadedUpdaterBase::StartWorker() {
- if (HttpClient_) {
- HttpClient_->ResetConnection();
- }
- Thread_ = MakeHolder<TThread>(WorkerWrap, this);
- Thread_->Start();
- Started_.Wait();
- IsStopped_ = false;
- }
-
- void TThreadedUpdaterBase::StopWorker() {
- Event_.Signal();
- if (Thread_) {
- Thread_.Reset();
- }
- }
-
- TKeepAliveHttpClient& TThreadedUpdaterBase::GetClient() const {
- if (!HttpClient_) {
- HttpClient_ = MakeHolder<TKeepAliveHttpClient>(TvmUrl_, TvmPort_, TvmSocketTimeout_, TvmConnectTimeout_);
- }
-
- return *HttpClient_;
- }
-
- void TThreadedUpdaterBase::LogDebug(const TString& msg) const {
- if (Logger_) {
- Logger_->Debug(msg);
- }
- }
-
- void TThreadedUpdaterBase::LogInfo(const TString& msg) const {
- if (Logger_) {
- Logger_->Info(msg);
- }
- }
-
- void TThreadedUpdaterBase::LogWarning(const TString& msg) const {
- if (Logger_) {
- Logger_->Warning(msg);
- }
- }
-
- void TThreadedUpdaterBase::LogError(const TString& msg) const {
- if (Logger_) {
- Logger_->Error(msg);
- }
- }
-
- void* TThreadedUpdaterBase::WorkerWrap(void* arg) {
- TThread::SetCurrentThreadName("TicketParserUpd");
- TThreadedUpdaterBase& this_ = *reinterpret_cast<TThreadedUpdaterBase*>(arg);
- this_.Started_.Signal();
- this_.LogDebug("Thread-worker started");
-
- while (true) {
- if (this_.Event_.WaitT(this_.WorkerAwakingPeriod_)) {
- break;
- }
-
- try {
- this_.Worker();
- this_.GetClient().ResetConnection();
- } catch (const std::exception& e) { // impossible now
- this_.LogError(TStringBuilder() << "Failed to generate new cache: " << e.what());
- }
- }
-
- this_.LogDebug("Thread-worker stopped");
- this_.IsStopped_ = true;
- return nullptr;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/threaded_updater.h b/library/cpp/tvmauth/client/misc/threaded_updater.h
deleted file mode 100644
index 783684ba3b..0000000000
--- a/library/cpp/tvmauth/client/misc/threaded_updater.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#pragma once
-
-#include "async_updater.h"
-#include "settings.h"
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/http/simple/http_client.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/ptr.h>
-#include <util/system/event.h>
-#include <util/system/thread.h>
-
-class TKeepAliveHttpClient;
-
-namespace NTvmAuth::NInternal {
- class TClientCaningKnife;
-}
-namespace NTvmAuth {
- class TThreadedUpdaterBase: public TAsyncUpdaterBase {
- public:
- TThreadedUpdaterBase(TDuration workerAwakingPeriod,
- TLoggerPtr logger,
- const TString& url,
- ui16 port,
- TDuration socketTimeout,
- TDuration connectTimeout);
- virtual ~TThreadedUpdaterBase();
-
- protected:
- void StartWorker();
- void StopWorker();
-
- virtual void Worker() {
- }
-
- TKeepAliveHttpClient& GetClient() const;
-
- void LogDebug(const TString& msg) const;
- void LogInfo(const TString& msg) const;
- void LogWarning(const TString& msg) const;
- void LogError(const TString& msg) const;
-
- protected:
- TDuration WorkerAwakingPeriod_;
-
- const TLoggerPtr Logger_;
-
- protected:
- const TString TvmUrl_;
-
- private:
- static void* WorkerWrap(void* arg);
-
- void StartTvmClientStopping() const override {
- Event_.Signal();
- }
-
- bool IsTvmClientStopped() const override {
- return IsStopped_;
- }
-
- private:
- mutable THolder<TKeepAliveHttpClient> HttpClient_;
-
- const ui32 TvmPort_;
- const TDuration TvmSocketTimeout_;
- const TDuration TvmConnectTimeout_;
-
- mutable TAutoEvent Event_;
- mutable TAutoEvent Started_;
- std::atomic_bool IsStopped_;
- THolder<TThread> Thread_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/meta_info.cpp b/library/cpp/tvmauth/client/misc/tool/meta_info.cpp
deleted file mode 100644
index 9a0ae228fe..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/meta_info.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-#include "meta_info.h"
-
-#include <library/cpp/json/json_reader.h>
-
-#include <util/string/builder.h>
-
-namespace NTvmAuth::NTvmTool {
- TString TMetaInfo::TConfig::ToString() const {
- TStringStream s;
- s << "self_tvm_id=" << SelfTvmId << ", "
- << "bb_env=" << BbEnv << ", "
- << "idm_slug=" << (IdmSlug ? IdmSlug : "<NULL>") << ", "
- << "dsts=[";
-
- for (const auto& pair : DstAliases) {
- s << "(" << pair.first << ":" << pair.second << ")";
- }
-
- s << "]";
-
- return std::move(s.Str());
- }
-
- TMetaInfo::TMetaInfo(TLoggerPtr logger)
- : Logger_(std::move(logger))
- {
- }
-
- TMetaInfo::TConfigPtr TMetaInfo::Init(TKeepAliveHttpClient& client,
- const TClientSettings& settings) {
- ApplySettings(settings);
-
- TryPing(client);
- const TString metaString = Fetch(client);
- if (Logger_) {
- TStringStream s;
- s << "Meta info fetched from " << settings.GetHostname() << ":" << settings.GetPort();
- Logger_->Debug(s.Str());
- }
-
- try {
- Config_.Set(ParseMetaString(metaString, SelfAlias_));
- } catch (const yexception& e) {
- ythrow TNonRetriableException() << "Malformed json from tvmtool: " << e.what();
- }
- TConfigPtr cfg = Config_.Get();
- Y_ENSURE_EX(cfg, TNonRetriableException() << "Alias '" << SelfAlias_ << "' not found in meta info");
-
- if (Logger_) {
- Logger_->Info("Meta: " + cfg->ToString());
- }
-
- return cfg;
- }
-
- TString TMetaInfo::GetRequestForTickets(const TConfig& config) {
- Y_ENSURE(!config.DstAliases.empty());
-
- TStringStream s;
- s << "/tvm/tickets"
- << "?src=" << config.SelfTvmId
- << "&dsts=";
-
- for (const auto& pair : config.DstAliases) {
- s << pair.second << ","; // avoid aliases - url-encoding required
- }
- s.Str().pop_back();
-
- return s.Str();
- }
-
- bool TMetaInfo::TryUpdateConfig(TKeepAliveHttpClient& client) {
- const TString metaString = Fetch(client);
-
- TConfigPtr config;
- try {
- config = ParseMetaString(metaString, SelfAlias_);
- } catch (const yexception& e) {
- ythrow TNonRetriableException() << "Malformed json from tvmtool: " << e.what();
- }
- Y_ENSURE_EX(config, TNonRetriableException() << "Alias '" << SelfAlias_ << "' not found in meta info");
-
- TConfigPtr oldConfig = Config_.Get();
- if (*config == *oldConfig) {
- return false;
- }
-
- if (Logger_) {
- Logger_->Info(TStringBuilder()
- << "Meta was updated. Old: (" << oldConfig->ToString()
- << "). New: (" << config->ToString() << ")");
- }
-
- Config_ = config;
- return true;
- }
-
- void TMetaInfo::TryPing(TKeepAliveHttpClient& client) {
- try {
- TStringStream s;
- TKeepAliveHttpClient::THttpCode code = client.DoGet("/tvm/ping", &s);
- if (code < 200 || 300 <= code) {
- throw yexception() << "(" << code << ") " << s.Str();
- }
- } catch (const std::exception& e) {
- ythrow TNonRetriableException() << "Failed to connect to tvmtool: " << e.what();
- }
- }
-
- TString TMetaInfo::Fetch(TKeepAliveHttpClient& client) const {
- TStringStream res;
- TKeepAliveHttpClient::THttpCode code;
- try {
- code = client.DoGet("/tvm/private_api/__meta__", &res, AuthHeader_);
- } catch (const std::exception& e) {
- ythrow TRetriableException() << "Failed to fetch meta data from tvmtool: " << e.what();
- }
-
- if (code != 200) {
- Y_ENSURE_EX(code != 404,
- TNonRetriableException() << "Library does not support so old tvmtool. You need tvmtool>=1.1.0");
-
- TStringStream err;
- err << "Failed to fetch meta from tvmtool: " << client.GetHost() << ":" << client.GetPort()
- << " (" << code << "): " << res.Str();
- Y_ENSURE_EX(!(500 <= code && code < 600), TRetriableException() << err.Str());
- ythrow TNonRetriableException() << err.Str();
- }
-
- return res.Str();
- }
-
- static TMetaInfo::TDstAliases::value_type ParsePair(const NJson::TJsonValue& val, const TString& meta) {
- NJson::TJsonValue jAlias;
- Y_ENSURE(val.GetValue("alias", &jAlias), meta);
- Y_ENSURE(jAlias.IsString(), meta);
-
- NJson::TJsonValue jClientId;
- Y_ENSURE(val.GetValue("client_id", &jClientId), meta);
- Y_ENSURE(jClientId.IsInteger(), meta);
-
- return {jAlias.GetString(), jClientId.GetInteger()};
- }
-
- TMetaInfo::TConfigPtr TMetaInfo::ParseMetaString(const TString& meta, const TString& self) {
- NJson::TJsonValue jDoc;
- Y_ENSURE(NJson::ReadJsonTree(meta, &jDoc), meta);
-
- NJson::TJsonValue jEnv;
- Y_ENSURE(jDoc.GetValue("bb_env", &jEnv), meta);
-
- NJson::TJsonValue jTenants;
- Y_ENSURE(jDoc.GetValue("tenants", &jTenants), meta);
- Y_ENSURE(jTenants.IsArray(), meta);
-
- for (const NJson::TJsonValue& jTen : jTenants.GetArray()) {
- NJson::TJsonValue jSelf;
- Y_ENSURE(jTen.GetValue("self", &jSelf), meta);
- auto selfPair = ParsePair(jSelf, meta);
- if (selfPair.first != self) {
- continue;
- }
-
- TConfigPtr config = std::make_shared<TConfig>();
- config->SelfTvmId = selfPair.second;
- config->BbEnv = BbEnvFromString(jEnv.GetString(), meta);
-
- {
- NJson::TJsonValue jSlug;
- if (jTen.GetValue("idm_slug", &jSlug)) {
- config->IdmSlug = jSlug.GetString();
- }
- }
-
- NJson::TJsonValue jDsts;
- Y_ENSURE(jTen.GetValue("dsts", &jDsts), meta);
- Y_ENSURE(jDsts.IsArray(), meta);
- for (const NJson::TJsonValue& jDst : jDsts.GetArray()) {
- config->DstAliases.insert(ParsePair(jDst, meta));
- }
-
- return config;
- }
-
- return {};
- }
-
- void TMetaInfo::ApplySettings(const TClientSettings& settings) {
- AuthHeader_ = {{"Authorization", settings.GetAuthToken()}};
- SelfAlias_ = settings.GetSelfAlias();
- }
-
- EBlackboxEnv TMetaInfo::BbEnvFromString(const TString& env, const TString& meta) {
- if (env == "Prod") {
- return EBlackboxEnv::Prod;
- } else if (env == "Test") {
- return EBlackboxEnv::Test;
- } else if (env == "ProdYaTeam") {
- return EBlackboxEnv::ProdYateam;
- } else if (env == "TestYaTeam") {
- return EBlackboxEnv::TestYateam;
- } else if (env == "Stress") {
- return EBlackboxEnv::Stress;
- }
-
- ythrow yexception() << "'bb_env'=='" << env << "'. " << meta;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/meta_info.h b/library/cpp/tvmauth/client/misc/tool/meta_info.h
deleted file mode 100644
index 9dd4f0dbf8..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/meta_info.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#pragma once
-
-#include "settings.h"
-
-#include <library/cpp/tvmauth/client/misc/utils.h>
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/http/simple/http_client.h>
-
-namespace NTvmAuth::NTvmTool {
- class TMetaInfo {
- public:
- using TDstAliases = THashMap<TClientSettings::TAlias, TTvmId>;
-
- struct TConfig {
- TTvmId SelfTvmId = 0;
- EBlackboxEnv BbEnv = EBlackboxEnv::Prod;
- TString IdmSlug;
- TDstAliases DstAliases;
-
- bool AreTicketsRequired() const {
- return !DstAliases.empty();
- }
-
- TString ToString() const;
-
- bool operator==(const TConfig& c) const {
- return SelfTvmId == c.SelfTvmId &&
- BbEnv == c.BbEnv &&
- IdmSlug == c.IdmSlug &&
- DstAliases == c.DstAliases;
- }
- };
- using TConfigPtr = std::shared_ptr<TConfig>;
-
- public:
- TMetaInfo(TLoggerPtr logger);
-
- TConfigPtr Init(TKeepAliveHttpClient& client,
- const TClientSettings& settings);
-
- static TString GetRequestForTickets(const TMetaInfo::TConfig& config);
-
- const TKeepAliveHttpClient::THeaders& GetAuthHeader() const {
- return AuthHeader_;
- }
-
- TConfigPtr GetConfig() const {
- return Config_.Get();
- }
-
- bool TryUpdateConfig(TKeepAliveHttpClient& client);
-
- protected:
- void TryPing(TKeepAliveHttpClient& client);
- TString Fetch(TKeepAliveHttpClient& client) const;
- static TConfigPtr ParseMetaString(const TString& meta, const TString& self);
- void ApplySettings(const TClientSettings& settings);
- static EBlackboxEnv BbEnvFromString(const TString& env, const TString& meta);
-
- protected:
- NUtils::TProtectedValue<TConfigPtr> Config_;
- TKeepAliveHttpClient::THeaders AuthHeader_;
-
- TLoggerPtr Logger_;
- TString SelfAlias_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/roles_fetcher.cpp b/library/cpp/tvmauth/client/misc/tool/roles_fetcher.cpp
deleted file mode 100644
index 05b0856edc..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/roles_fetcher.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "roles_fetcher.h"
-
-#include <library/cpp/tvmauth/client/misc/roles/parser.h>
-
-#include <library/cpp/http/misc/httpcodes.h>
-#include <library/cpp/string_utils/quote/quote.h>
-
-#include <util/string/builder.h>
-#include <util/string/join.h>
-
-namespace NTvmAuth::NTvmTool {
- TRolesFetcher::TRolesFetcher(const TRolesFetcherSettings& settings, TLoggerPtr logger)
- : Settings_(settings)
- , Logger_(std::move(logger))
- {
- }
-
- bool TRolesFetcher::IsTimeToUpdate(TDuration sinceUpdate) const {
- return Settings_.UpdatePeriod < sinceUpdate;
- }
-
- bool TRolesFetcher::ShouldWarn(TDuration sinceUpdate) const {
- return Settings_.WarnPeriod < sinceUpdate;
- }
-
- bool TRolesFetcher::AreRolesOk() const {
- return bool(GetCurrentRoles());
- }
-
- NUtils::TFetchResult TRolesFetcher::FetchActualRoles(const TKeepAliveHttpClient::THeaders& authHeader,
- TKeepAliveHttpClient& client) const {
- const TRequest req = CreateRequest(authHeader);
-
- TStringStream out;
- THttpHeaders outHeaders;
-
- TKeepAliveHttpClient::THttpCode code = client.DoGet(
- req.Url,
- &out,
- req.Headers,
- &outHeaders);
-
- return {code, std::move(outHeaders), "/v2/roles", out.Str(), {}};
- }
-
- void TRolesFetcher::Update(NUtils::TFetchResult&& fetchResult) {
- if (fetchResult.Code == HTTP_NOT_MODIFIED) {
- Y_ENSURE(CurrentRoles_.Get(),
- "tvmtool did not return any roles because current roles are actual,"
- " but there are no roles in memory - this should never happen");
- return;
- }
-
- Y_ENSURE(fetchResult.Code == HTTP_OK,
- "Unexpected code from tvmtool: " << fetchResult.Code << ". " << fetchResult.Response);
-
- CurrentRoles_.Set(NRoles::TParser::Parse(std::make_shared<TString>(std::move(fetchResult.Response))));
-
- Logger_->Debug(
- TStringBuilder() << "Succeed to update roles with revision "
- << CurrentRoles_.Get()->GetMeta().Revision);
- }
-
- NTvmAuth::NRoles::TRolesPtr TRolesFetcher::GetCurrentRoles() const {
- return CurrentRoles_.Get();
- }
-
- TRolesFetcher::TRequest TRolesFetcher::CreateRequest(const TKeepAliveHttpClient::THeaders& authHeader) const {
- TRequest request{
- .Url = "/v2/roles?self=" + CGIEscapeRet(Settings_.SelfAlias),
- .Headers = authHeader,
- };
-
- NRoles::TRolesPtr roles = CurrentRoles_.Get();
- if (roles) {
- request.Headers.emplace(IfNoneMatch_, Join("", "\"", roles->GetMeta().Revision, "\""));
- }
-
- return request;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/roles_fetcher.h b/library/cpp/tvmauth/client/misc/tool/roles_fetcher.h
deleted file mode 100644
index 8c60b59610..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/roles_fetcher.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/misc/fetch_result.h>
-#include <library/cpp/tvmauth/client/misc/utils.h>
-#include <library/cpp/tvmauth/client/misc/roles/roles.h>
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/string.h>
-
-namespace NTvmAuth::NTvmTool {
- struct TRolesFetcherSettings {
- TString SelfAlias;
- TDuration UpdatePeriod = TDuration::Minutes(1);
- TDuration WarnPeriod = TDuration::Minutes(20);
- };
-
- class TRolesFetcher {
- public:
- TRolesFetcher(const TRolesFetcherSettings& settings, TLoggerPtr logger);
-
- bool IsTimeToUpdate(TDuration sinceUpdate) const;
- bool ShouldWarn(TDuration sinceUpdate) const;
- bool AreRolesOk() const;
-
- NUtils::TFetchResult FetchActualRoles(const TKeepAliveHttpClient::THeaders& authHeader,
- TKeepAliveHttpClient& client) const;
- void Update(NUtils::TFetchResult&& fetchResult);
-
- NTvmAuth::NRoles::TRolesPtr GetCurrentRoles() const;
-
- protected:
- struct TRequest {
- TString Url;
- TKeepAliveHttpClient::THeaders Headers;
- };
-
- protected:
- TRequest CreateRequest(const TKeepAliveHttpClient::THeaders& authHeader) const;
-
- private:
- const TRolesFetcherSettings Settings_;
- const TLoggerPtr Logger_;
- const TString IfNoneMatch_ = "If-None-Match";
-
- NUtils::TProtectedValue<NTvmAuth::NRoles::TRolesPtr> CurrentRoles_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/settings.cpp b/library/cpp/tvmauth/client/misc/tool/settings.cpp
deleted file mode 100644
index 894501f19d..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/settings.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-#include "settings.h"
-
-#include <library/cpp/string_utils/url/url.h>
-
-#include <util/system/env.h>
-
-namespace NTvmAuth::NTvmTool {
- TClientSettings::TClientSettings(const TAlias& selfAias)
- : SelfAias_(selfAias)
- , Hostname_("localhost")
- , Port_(1)
- , SocketTimeout_(TDuration::Seconds(5))
- , ConnectTimeout_(TDuration::Seconds(30))
- {
- AuthToken_ = GetEnv("TVMTOOL_LOCAL_AUTHTOKEN");
- if (!AuthToken_) {
- AuthToken_ = GetEnv("QLOUD_TVM_TOKEN");
- }
- TStringBuf auth(AuthToken_);
- FixSpaces(auth);
- AuthToken_ = auth;
-
- const TString url = GetEnv("DEPLOY_TVM_TOOL_URL");
- if (url) {
- TStringBuf scheme, host;
- TryGetSchemeHostAndPort(url, scheme, host, Port_);
- }
-
- Y_ENSURE_EX(SelfAias_, TBrokenTvmClientSettings() << "Alias for your TVM client cannot be empty");
- }
-
- void TClientSettings::FixSpaces(TStringBuf& str) {
- while (str && isspace(str.back())) {
- str.Chop(1);
- }
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/settings.h b/library/cpp/tvmauth/client/misc/tool/settings.h
deleted file mode 100644
index 63255ed090..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/settings.h
+++ /dev/null
@@ -1,137 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/misc/settings.h>
-
-#include <library/cpp/tvmauth/client/exception.h>
-
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/maybe.h>
-
-namespace NTvmAuth::NTvmTool {
- /**
- * Uses local http-interface to get state: http://localhost/tvm/.
- * This interface can be provided with tvmtool (local daemon) or Qloud/YP (local http api in container).
- * See more: https://wiki.yandex-team.ru/passport/tvm2/qloud/.
- *
- * Most part of settings will be fetched from tvmtool on start of client.
- * You need to use aliases for TVM-clients (src and dst) which you specified in tvmtool or Qloud/YP interface
- */
- class TClientSettings: public NTvmAuth::TClientSettings {
- public:
- /*!
- * Sets default values:
- * - hostname == "localhost"
- * - port detected with env["DEPLOY_TVM_TOOL_URL"] (provided with Yandex.Deploy),
- * otherwise port == 1 (it is ok for Qloud)
- * - authToken: env["TVMTOOL_LOCAL_AUTHTOKEN"] (provided with Yandex.Deploy),
- * otherwise env["QLOUD_TVM_TOKEN"] (provided with Qloud)
- *
- * AuthToken is protection from SSRF.
- *
- * @param selfAias - alias for your TVM client, which you specified in tvmtool or YD interface
- */
- TClientSettings(const TAlias& selfAias);
-
- /*!
- * Look at comment for ctor
- * @param port
- */
- TClientSettings& SetPort(ui16 port) {
- Port_ = port;
- return *this;
- }
-
- /*!
- * Default value: hostname == "localhost"
- * @param hostname
- */
- TClientSettings& SetHostname(const TString& hostname) {
- Y_ENSURE_EX(hostname, TBrokenTvmClientSettings() << "Hostname cannot be empty");
- Hostname_ = hostname;
- return *this;
- }
-
- TClientSettings& SetSocketTimeout(TDuration socketTimeout) {
- SocketTimeout_ = socketTimeout;
- return *this;
- }
-
- TClientSettings& SetConnectTimeout(TDuration connectTimeout) {
- ConnectTimeout_ = connectTimeout;
- return *this;
- }
-
- /*!
- * Look at comment for ctor
- * @param token
- */
- TClientSettings& SetAuthToken(TStringBuf token) {
- FixSpaces(token);
- Y_ENSURE_EX(token, TBrokenTvmClientSettings() << "Auth token cannot be empty");
- AuthToken_ = token;
- return *this;
- }
-
- /*!
- * Blackbox environmet is provided by tvmtool for client.
- * You can override it for your purpose with limitations:
- * (env from tvmtool) -> (override)
- * - Prod/ProdYateam -> Prod/ProdYateam
- * - Test/TestYateam -> Test/TestYateam
- * - Stress -> Stress
- *
- * You can contact tvm-dev@yandex-team.ru if limitations are too strict
- * @param env
- */
- TClientSettings& OverrideBlackboxEnv(EBlackboxEnv env) {
- BbEnv_ = env;
- return *this;
- }
-
- public: // for TAsyncUpdaterBase
- const TAlias& GetSelfAlias() const {
- return SelfAias_;
- }
-
- const TString& GetHostname() const {
- return Hostname_;
- }
-
- ui16 GetPort() const {
- return Port_;
- }
-
- TDuration GetSocketTimeout() const {
- return SocketTimeout_;
- }
-
- TDuration GetConnectTimeout() const {
- return ConnectTimeout_;
- }
-
- const TString& GetAuthToken() const {
- Y_ENSURE_EX(AuthToken_, TBrokenTvmClientSettings()
- << "Auth token cannot be empty. "
- << "Env 'TVMTOOL_LOCAL_AUTHTOKEN' and 'QLOUD_TVM_TOKEN' are empty.");
- return AuthToken_;
- }
-
- TMaybe<EBlackboxEnv> GetOverridedBlackboxEnv() const {
- return BbEnv_;
- }
-
- private:
- void FixSpaces(TStringBuf& str);
-
- private:
- TAlias SelfAias_;
- TString Hostname_;
- ui16 Port_;
- TDuration SocketTimeout_;
- TDuration ConnectTimeout_;
- TString AuthToken_;
- TMaybe<EBlackboxEnv> BbEnv_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/threaded_updater.cpp b/library/cpp/tvmauth/client/misc/tool/threaded_updater.cpp
deleted file mode 100644
index 35bbe4f617..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/threaded_updater.cpp
+++ /dev/null
@@ -1,370 +0,0 @@
-#include "threaded_updater.h"
-
-#include <library/cpp/tvmauth/client/misc/utils.h>
-
-#include <library/cpp/json/json_reader.h>
-
-#include <util/generic/hash_set.h>
-#include <util/stream/str.h>
-#include <util/string/ascii.h>
-#include <util/string/builder.h>
-#include <util/string/cast.h>
-
-namespace NTvmAuth::NTvmTool {
- TAsyncUpdaterPtr TThreadedUpdater::Create(const TClientSettings& settings, TLoggerPtr logger) {
- Y_ENSURE_EX(logger, TNonRetriableException() << "Logger is required");
- THolder<TThreadedUpdater> p(new TThreadedUpdater(
- settings.GetHostname(),
- settings.GetPort(),
- settings.GetSocketTimeout(),
- settings.GetConnectTimeout(),
- std::move(logger)));
- p->Init(settings);
- p->StartWorker();
- return p.Release();
- }
-
- TThreadedUpdater::~TThreadedUpdater() {
- StopWorker(); // Required here to avoid using of deleted members
- }
-
- TClientStatus TThreadedUpdater::GetStatus() const {
- const TClientStatus::ECode state = GetState();
- return TClientStatus(state, GetLastError(state == TClientStatus::Ok));
- }
-
- NRoles::TRolesPtr TThreadedUpdater::GetRoles() const {
- Y_ENSURE_EX(RolesFetcher_,
- TBrokenTvmClientSettings() << "Roles were not configured in settings");
- return RolesFetcher_->GetCurrentRoles();
- }
-
- TClientStatus::ECode TThreadedUpdater::GetState() const {
- const TInstant now = TInstant::Now();
- const TMetaInfo::TConfigPtr config = MetaInfo_.GetConfig();
-
- if ((config->AreTicketsRequired() && AreServiceTicketsInvalid(now)) || ArePublicKeysInvalid(now)) {
- return TClientStatus::Error;
- }
-
- if (config->AreTicketsRequired()) {
- if (!GetCachedServiceTickets() || config->DstAliases.size() > GetCachedServiceTickets()->TicketsByAlias.size()) {
- return TClientStatus::Error;
- }
- }
-
- const TDuration st = now - GetUpdateTimeOfServiceTickets();
- const TDuration pk = now - GetUpdateTimeOfPublicKeys();
-
- if ((config->AreTicketsRequired() && st > ServiceTicketsDurations_.Expiring) || pk > PublicKeysDurations_.Expiring) {
- return TClientStatus::Warning;
- }
-
- if (RolesFetcher_ && RolesFetcher_->ShouldWarn(now - GetUpdateTimeOfRoles())) {
- return TClientStatus::Warning;
- }
-
- if (IsConfigWarnTime()) {
- return TClientStatus::Warning;
- }
-
- return TClientStatus::Ok;
- }
-
- TThreadedUpdater::TThreadedUpdater(const TString& host, ui16 port, TDuration socketTimeout, TDuration connectTimeout, TLoggerPtr logger)
- : TThreadedUpdaterBase(TDuration::Seconds(5), logger, host, port, socketTimeout, connectTimeout)
- , MetaInfo_(logger)
- , ConfigWarnDelay_(TDuration::Seconds(30))
- {
- ServiceTicketsDurations_.RefreshPeriod = TDuration::Minutes(10);
- PublicKeysDurations_.RefreshPeriod = TDuration::Minutes(10);
- }
-
- void TThreadedUpdater::Init(const TClientSettings& settings) {
- const TMetaInfo::TConfigPtr config = MetaInfo_.Init(GetClient(), settings);
- LastVisitForConfig_ = TInstant::Now();
-
- SetBbEnv(config->BbEnv, settings.GetOverridedBlackboxEnv());
- if (settings.GetOverridedBlackboxEnv()) {
- LogInfo(TStringBuilder()
- << "Meta: override blackbox env: " << config->BbEnv
- << "->" << *settings.GetOverridedBlackboxEnv());
- }
-
- if (config->IdmSlug) {
- RolesFetcher_ = std::make_unique<TRolesFetcher>(
- TRolesFetcherSettings{
- .SelfAlias = settings.GetSelfAlias(),
- },
- Logger_);
- }
-
- ui8 tries = 3;
- do {
- UpdateState();
- } while (!IsEverythingOk(*config) && --tries > 0);
-
- if (!IsEverythingOk(*config)) {
- ThrowLastError();
- }
- }
-
- void TThreadedUpdater::UpdateState() {
- bool wasUpdated = false;
- try {
- wasUpdated = MetaInfo_.TryUpdateConfig(GetClient());
- LastVisitForConfig_ = TInstant::Now();
- ClearError(EScope::TvmtoolConfig);
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::TvmtoolConfig, e.what());
- LogWarning(TStringBuilder() << "Error while fetching of tvmtool config: " << e.what());
- }
- if (IsConfigWarnTime()) {
- LogError(TStringBuilder() << "Tvmtool config have not been refreshed for too long period");
- }
-
- TMetaInfo::TConfigPtr config = MetaInfo_.GetConfig();
-
- if (wasUpdated || IsTimeToUpdateServiceTickets(*config, LastVisitForServiceTickets_)) {
- try {
- const TInstant updateTime = UpdateServiceTickets(*config);
- SetUpdateTimeOfServiceTickets(updateTime);
- LastVisitForServiceTickets_ = TInstant::Now();
-
- if (AreServiceTicketsOk(*config)) {
- ClearError(EScope::ServiceTickets);
- }
- LogDebug(TStringBuilder() << "Tickets fetched from tvmtool: " << updateTime);
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::ServiceTickets, e.what());
- LogWarning(TStringBuilder() << "Error while fetching of tickets: " << e.what());
- }
-
- if (TInstant::Now() - GetUpdateTimeOfServiceTickets() > ServiceTicketsDurations_.Expiring) {
- LogError("Service tickets have not been refreshed for too long period");
- }
- }
-
- if (wasUpdated || IsTimeToUpdatePublicKeys(LastVisitForPublicKeys_)) {
- try {
- const TInstant updateTime = UpdateKeys(*config);
- SetUpdateTimeOfPublicKeys(updateTime);
- LastVisitForPublicKeys_ = TInstant::Now();
-
- if (ArePublicKeysOk()) {
- ClearError(EScope::PublicKeys);
- }
- LogDebug(TStringBuilder() << "Public keys fetched from tvmtool: " << updateTime);
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::PublicKeys, e.what());
- LogWarning(TStringBuilder() << "Error while fetching of public keys: " << e.what());
- }
-
- if (TInstant::Now() - GetUpdateTimeOfPublicKeys() > PublicKeysDurations_.Expiring) {
- LogError("Public keys have not been refreshed for too long period");
- }
- }
-
- if (RolesFetcher_ && (wasUpdated || RolesFetcher_->IsTimeToUpdate(TInstant::Now() - GetUpdateTimeOfRoles()))) {
- try {
- RolesFetcher_->Update(RolesFetcher_->FetchActualRoles(MetaInfo_.GetAuthHeader(), GetClient()));
- SetUpdateTimeOfRoles(TInstant::Now());
-
- if (RolesFetcher_->AreRolesOk()) {
- ClearError(EScope::Roles);
- }
- } catch (const std::exception& e) {
- ProcessError(EType::Retriable, EScope::Roles, e.what());
- LogWarning(TStringBuilder() << "Failed to update roles: " << e.what());
- }
-
- if (RolesFetcher_->ShouldWarn(TInstant::Now() - GetUpdateTimeOfRoles())) {
- LogError("Roles have not been refreshed for too long period");
- }
- }
- }
-
- TInstant TThreadedUpdater::UpdateServiceTickets(const TMetaInfo::TConfig& config) {
- const std::pair<TString, TInstant> tickets = FetchServiceTickets(config);
-
- if (TInstant::Now() - tickets.second >= ServiceTicketsDurations_.Invalid) {
- throw yexception() << "Service tickets are too old: " << tickets.second;
- }
-
- TPairTicketsErrors p = ParseFetchTicketsResponse(tickets.first, config.DstAliases);
- SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(std::move(p.Tickets),
- std::move(p.Errors),
- config.DstAliases));
- return tickets.second;
- }
-
- std::pair<TString, TInstant> TThreadedUpdater::FetchServiceTickets(const TMetaInfo::TConfig& config) const {
- TStringStream s;
- THttpHeaders headers;
-
- const TString request = TMetaInfo::GetRequestForTickets(config);
- auto code = GetClient().DoGet(request, &s, MetaInfo_.GetAuthHeader(), &headers);
- Y_ENSURE(code == 200, ProcessHttpError(EScope::ServiceTickets, request, code, s.Str()));
-
- return {s.Str(), GetBirthTimeFromResponse(headers, "tickets")};
- }
-
- static THashSet<TTvmId> GetAllTvmIds(const TMetaInfo::TDstAliases& dsts) {
- THashSet<TTvmId> res;
- res.reserve(dsts.size());
-
- for (const auto& pair : dsts) {
- res.insert(pair.second);
- }
-
- return res;
- }
-
- TAsyncUpdaterBase::TPairTicketsErrors TThreadedUpdater::ParseFetchTicketsResponse(const TString& resp,
- const TMetaInfo::TDstAliases& dsts) const {
- const THashSet<TTvmId> allTvmIds = GetAllTvmIds(dsts);
-
- TServiceTickets::TMapIdStr tickets;
- TServiceTickets::TMapIdStr errors;
-
- auto procErr = [this](const TString& msg) {
- ProcessError(EType::NonRetriable, EScope::ServiceTickets, msg);
- LogError(msg);
- };
-
- NJson::TJsonValue doc;
- Y_ENSURE(NJson::ReadJsonTree(resp, &doc), "Invalid json from tvmtool: " << resp);
-
- for (const auto& pair : doc.GetMap()) {
- NJson::TJsonValue tvmId;
- unsigned long long tvmIdNum = 0;
-
- if (!pair.second.GetValue("tvm_id", &tvmId) ||
- !tvmId.GetUInteger(&tvmIdNum)) {
- procErr(TStringBuilder()
- << "Failed to get 'tvm_id' from key, should never happend '"
- << pair.first << "': " << resp);
- continue;
- }
-
- if (!allTvmIds.contains(tvmIdNum)) {
- continue;
- }
-
- NJson::TJsonValue val;
- if (!pair.second.GetValue("ticket", &val)) {
- TString err;
- if (pair.second.GetValue("error", &val)) {
- err = val.GetString();
- } else {
- err = "Failed to get 'ticket' and 'error', should never happend: " + pair.first;
- }
-
- procErr(TStringBuilder()
- << "Failed to get ServiceTicket for " << pair.first
- << " (" << tvmIdNum << "): " << err);
-
- errors.insert({tvmIdNum, std::move(err)});
- continue;
- }
-
- tickets.insert({tvmIdNum, val.GetString()});
- }
-
- // This work-around is required because of bug in old verions of tvmtool: PASSP-24829
- for (const auto& pair : dsts) {
- if (!tickets.contains(pair.second) && !errors.contains(pair.second)) {
- TString err = "Missing tvm_id in response, should never happend: " + pair.first;
-
- procErr(TStringBuilder()
- << "Failed to get ServiceTicket for " << pair.first
- << " (" << pair.second << "): " << err);
-
- errors.emplace(pair.second, std::move(err));
- }
- }
-
- return {std::move(tickets), std::move(errors)};
- }
-
- TInstant TThreadedUpdater::UpdateKeys(const TMetaInfo::TConfig& config) {
- const std::pair<TString, TInstant> keys = FetchPublicKeys();
-
- if (TInstant::Now() - keys.second >= PublicKeysDurations_.Invalid) {
- throw yexception() << "Public keys are too old: " << keys.second;
- }
-
- SetServiceContext(MakeIntrusiveConst<TServiceContext>(
- TServiceContext::CheckingFactory(config.SelfTvmId, keys.first)));
- SetUserContext(keys.first);
-
- return keys.second;
- }
-
- std::pair<TString, TInstant> TThreadedUpdater::FetchPublicKeys() const {
- TStringStream s;
- THttpHeaders headers;
-
- auto code = GetClient().DoGet("/tvm/keys", &s, MetaInfo_.GetAuthHeader(), &headers);
- Y_ENSURE(code == 200, ProcessHttpError(EScope::PublicKeys, "/tvm/keys", code, s.Str()));
-
- return {s.Str(), GetBirthTimeFromResponse(headers, "public keys")};
- }
-
- TInstant TThreadedUpdater::GetBirthTimeFromResponse(const THttpHeaders& headers, TStringBuf errMsg) {
- auto it = std::find_if(headers.begin(),
- headers.end(),
- [](const THttpInputHeader& h) {
- return AsciiEqualsIgnoreCase(h.Name(), "X-Ya-Tvmtool-Data-Birthtime");
- });
- Y_ENSURE(it != headers.end(), "Failed to fetch bithtime of " << errMsg << " from tvmtool");
-
- ui64 time = 0;
- Y_ENSURE(TryIntFromString<10>(it->Value(), time),
- "Bithtime of " << errMsg << " from tvmtool must be unixtime. Got: " << it->Value());
-
- return TInstant::Seconds(time);
- }
-
- bool TThreadedUpdater::IsTimeToUpdateServiceTickets(const TMetaInfo::TConfig& config,
- TInstant lastUpdate) const {
- return config.AreTicketsRequired() &&
- TInstant::Now() - lastUpdate > ServiceTicketsDurations_.RefreshPeriod;
- }
-
- bool TThreadedUpdater::IsTimeToUpdatePublicKeys(TInstant lastUpdate) const {
- return TInstant::Now() - lastUpdate > PublicKeysDurations_.RefreshPeriod;
- }
-
- bool TThreadedUpdater::IsEverythingOk(const TMetaInfo::TConfig& config) const {
- if (RolesFetcher_ && !RolesFetcher_->AreRolesOk()) {
- return false;
- }
- return AreServiceTicketsOk(config) && ArePublicKeysOk();
- }
-
- bool TThreadedUpdater::AreServiceTicketsOk(const TMetaInfo::TConfig& config) const {
- return AreServiceTicketsOk(config.DstAliases.size());
- }
-
- bool TThreadedUpdater::AreServiceTicketsOk(size_t requiredCount) const {
- if (requiredCount == 0) {
- return true;
- }
-
- auto c = GetCachedServiceTickets();
- return c && c->TicketsByAlias.size() == requiredCount;
- }
-
- bool TThreadedUpdater::ArePublicKeysOk() const {
- return GetCachedServiceContext() && GetCachedUserContext();
- }
-
- bool TThreadedUpdater::IsConfigWarnTime() const {
- return LastVisitForConfig_ + ConfigWarnDelay_ < TInstant::Now();
- }
-
- void TThreadedUpdater::Worker() {
- UpdateState();
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/tool/threaded_updater.h b/library/cpp/tvmauth/client/misc/tool/threaded_updater.h
deleted file mode 100644
index 57f97f5442..0000000000
--- a/library/cpp/tvmauth/client/misc/tool/threaded_updater.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#pragma once
-
-#include "meta_info.h"
-#include "roles_fetcher.h"
-
-#include <library/cpp/tvmauth/client/misc/async_updater.h>
-#include <library/cpp/tvmauth/client/misc/threaded_updater.h>
-
-#include <atomic>
-
-namespace NTvmAuth::NTvmTool {
- class TThreadedUpdater: public TThreadedUpdaterBase {
- public:
- static TAsyncUpdaterPtr Create(const TClientSettings& settings, TLoggerPtr logger);
- ~TThreadedUpdater();
-
- TClientStatus GetStatus() const override;
- NRoles::TRolesPtr GetRoles() const override;
-
- protected: // for tests
- TClientStatus::ECode GetState() const;
-
- TThreadedUpdater(const TString& host, ui16 port, TDuration socketTimeout, TDuration connectTimeout, TLoggerPtr logger);
-
- void Init(const TClientSettings& settings);
- void UpdateState();
-
- TInstant UpdateServiceTickets(const TMetaInfo::TConfig& config);
- std::pair<TString, TInstant> FetchServiceTickets(const TMetaInfo::TConfig& config) const;
- TPairTicketsErrors ParseFetchTicketsResponse(const TString& resp,
- const TMetaInfo::TDstAliases& dsts) const;
-
- TInstant UpdateKeys(const TMetaInfo::TConfig& config);
- std::pair<TString, TInstant> FetchPublicKeys() const;
-
- static TInstant GetBirthTimeFromResponse(const THttpHeaders& headers, TStringBuf errMsg);
-
- bool IsTimeToUpdateServiceTickets(const TMetaInfo::TConfig& config, TInstant lastUpdate) const;
- bool IsTimeToUpdatePublicKeys(TInstant lastUpdate) const;
-
- bool IsEverythingOk(const TMetaInfo::TConfig& config) const;
- bool AreServiceTicketsOk(const TMetaInfo::TConfig& config) const;
- bool AreServiceTicketsOk(size_t requiredCount) const;
- bool ArePublicKeysOk() const;
- bool IsConfigWarnTime() const;
-
- private:
- void Worker() override;
-
- protected:
- TMetaInfo MetaInfo_;
- TInstant LastVisitForServiceTickets_;
- TInstant LastVisitForPublicKeys_;
- TInstant LastVisitForConfig_;
- TDuration ConfigWarnDelay_;
- std::unique_ptr<TRolesFetcher> RolesFetcher_;
- };
-}
diff --git a/library/cpp/tvmauth/client/misc/utils.cpp b/library/cpp/tvmauth/client/misc/utils.cpp
deleted file mode 100644
index a124c7b11c..0000000000
--- a/library/cpp/tvmauth/client/misc/utils.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "utils.h"
-
-#include <library/cpp/tvmauth/client/facade.h>
-
-#include <util/stream/format.h>
-
-namespace NTvmAuth::NInternal {
- void TClientCaningKnife::StartTvmClientStopping(TTvmClient* c) {
- if (c && c->Updater_) {
- c->Updater_->StartTvmClientStopping();
- }
- }
-
- bool TClientCaningKnife::IsTvmClientStopped(TTvmClient* c) {
- return c && c->Updater_ ? c->Updater_->IsTvmClientStopped() : true;
- }
-}
-
-namespace NTvmAuth::NUtils {
- TString ToHex(const TStringBuf s) {
- TStringStream res;
- res.Reserve(2 * s.size());
-
- for (char c : s) {
- res << Hex(c, HF_FULL);
- }
-
- return std::move(res.Str());
- }
-
- bool CheckBbEnvOverriding(EBlackboxEnv original, EBlackboxEnv override) noexcept {
- switch (original) {
- case EBlackboxEnv::Prod:
- case EBlackboxEnv::ProdYateam:
- return override == EBlackboxEnv::Prod || override == EBlackboxEnv::ProdYateam;
- case EBlackboxEnv::Test:
- return true;
- case EBlackboxEnv::TestYateam:
- return override == EBlackboxEnv::Test || override == EBlackboxEnv::TestYateam;
- case EBlackboxEnv::Stress:
- return override == EBlackboxEnv::Stress;
- }
-
- return false;
- }
-}
diff --git a/library/cpp/tvmauth/client/misc/utils.h b/library/cpp/tvmauth/client/misc/utils.h
deleted file mode 100644
index 1aa5e61bf1..0000000000
--- a/library/cpp/tvmauth/client/misc/utils.h
+++ /dev/null
@@ -1,95 +0,0 @@
-#pragma once
-
-#include "api/settings.h"
-#include "tool/settings.h"
-
-#include <util/string/cast.h>
-#include <util/system/spinlock.h>
-
-#include <optional>
-
-namespace NTvmAuth {
- class TTvmClient;
-}
-
-namespace NTvmAuth::NInternal {
- class TClientCaningKnife {
- public:
- static void StartTvmClientStopping(TTvmClient* c);
- static bool IsTvmClientStopped(TTvmClient* c);
- };
-}
-
-namespace NTvmAuth::NUtils {
- TString ToHex(const TStringBuf s);
-
- inline NTvmAuth::NTvmApi::TClientSettings::TDstMap ParseDstMap(TStringBuf dsts) {
- NTvmAuth::NTvmApi::TClientSettings::TDstMap res;
-
- while (dsts) {
- TStringBuf pair = dsts.NextTok(';');
- TStringBuf alias = pair.NextTok(':');
- res.insert(decltype(res)::value_type(
- alias,
- IntFromString<TTvmId, 10>(pair)));
- }
-
- return res;
- }
-
- inline NTvmAuth::NTvmApi::TClientSettings::TDstVector ParseDstVector(TStringBuf dsts) {
- NTvmAuth::NTvmApi::TClientSettings::TDstVector res;
-
- while (dsts) {
- res.push_back(IntFromString<TTvmId, 10>(dsts.NextTok(';')));
- }
-
- return res;
- }
-
- bool CheckBbEnvOverriding(EBlackboxEnv original, EBlackboxEnv override) noexcept;
-
- template <class T>
- class TProtectedValue {
- class TAssignOp {
- public:
- static void Assign(T& l, const T& r) {
- l = r;
- }
-
- template <typename U>
- static void Assign(std::shared_ptr<U>& l, std::shared_ptr<U>& r) {
- l.swap(r);
- }
-
- template <typename U>
- static void Assign(TIntrusiveConstPtr<U>& l, TIntrusiveConstPtr<U>& r) {
- l.Swap(r);
- }
- };
-
- public:
- TProtectedValue() = default;
-
- TProtectedValue(T value)
- : Value_(value)
- {
- }
-
- T Get() const {
- with_lock (Lock_) {
- return Value_;
- }
- }
-
- void Set(T o) {
- with_lock (Lock_) {
- TAssignOp::Assign(Value_, o);
- }
- }
-
- private:
- T Value_;
- mutable TAdaptiveLock Lock_;
- };
-}
diff --git a/library/cpp/tvmauth/client/mocked_updater.cpp b/library/cpp/tvmauth/client/mocked_updater.cpp
deleted file mode 100644
index 54f94bc92a..0000000000
--- a/library/cpp/tvmauth/client/mocked_updater.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-#include "mocked_updater.h"
-
-#include <library/cpp/tvmauth/unittest.h>
-
-namespace NTvmAuth {
- TMockedUpdater::TSettings TMockedUpdater::TSettings::CreateDeafult() {
- TMockedUpdater::TSettings res;
-
- res.SelfTvmId = 100500;
-
- res.Backends = {
- {
- /*.Alias_ = */ "my_dest",
- /*.Id_ = */ 42,
- /*.Value_ = */ "3:serv:CBAQ__________9_IgYIlJEGECo:O9-vbod_8czkKrpwJAZCI8UgOIhNr2xKPcS-LWALrVC224jga2nIT6vLiw6q3d6pAT60g9K7NB39LEmh7vMuePtUMjzuZuL-uJg17BsH2iTLCZSxDjWxbU9piA2T6u607jiSyiy-FI74pEPqkz7KKJ28aPsefuC1VUweGkYFzNY",
- },
- };
-
- res.BadBackends = {
- {
- /*.Alias_ = */ "my_bad_dest",
- /*.Id_ = */ 43,
- /*.Value_ = */ "Dst is not found",
- },
- };
-
- return res;
- }
-
- TMockedUpdater::TMockedUpdater(const TSettings& settings)
- : Roles_(settings.Roles)
- {
- SetServiceContext(MakeIntrusiveConst<TServiceContext>(TServiceContext::CheckingFactory(
- settings.SelfTvmId,
- NUnittest::TVMKNIFE_PUBLIC_KEYS)));
-
- SetBbEnv(settings.UserTicketEnv);
- SetUserContext(NUnittest::TVMKNIFE_PUBLIC_KEYS);
-
- TServiceTickets::TMapIdStr tickets, errors;
- TServiceTickets::TMapAliasId aliases;
-
- for (const TSettings::TTuple& t : settings.Backends) {
- tickets[t.Id] = t.Value;
- aliases[t.Alias] = t.Id;
- }
- for (const TSettings::TTuple& t : settings.BadBackends) {
- errors[t.Id] = t.Value;
- aliases[t.Alias] = t.Id;
- }
-
- SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- std::move(tickets),
- std::move(errors),
- std::move(aliases)));
-
- SetUpdateTimeOfPublicKeys(TInstant::Now());
- SetUpdateTimeOfServiceTickets(TInstant::Now());
- }
-}
diff --git a/library/cpp/tvmauth/client/mocked_updater.h b/library/cpp/tvmauth/client/mocked_updater.h
deleted file mode 100644
index f8a6394f5f..0000000000
--- a/library/cpp/tvmauth/client/mocked_updater.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#pragma once
-
-#include "misc/async_updater.h"
-
-namespace NTvmAuth {
- class TMockedUpdater: public TAsyncUpdaterBase {
- public:
- struct TSettings {
- struct TTuple {
- TClientSettings::TAlias Alias;
- TTvmId Id = 0;
- TString Value; // ticket or error
- };
-
- TTvmId SelfTvmId = 0;
- TVector<TTuple> Backends;
- TVector<TTuple> BadBackends;
- EBlackboxEnv UserTicketEnv = EBlackboxEnv::Test;
- NRoles::TRolesPtr Roles;
-
- static TSettings CreateDeafult();
- };
-
- TMockedUpdater(const TSettings& settings = TSettings::CreateDeafult());
-
- TClientStatus GetStatus() const override {
- return TClientStatus();
- }
-
- NRoles::TRolesPtr GetRoles() const override {
- return Roles_;
- }
-
- using TAsyncUpdaterBase::SetServiceContext;
- using TAsyncUpdaterBase::SetServiceTickets;
- using TAsyncUpdaterBase::SetUpdateTimeOfPublicKeys;
- using TAsyncUpdaterBase::SetUpdateTimeOfServiceTickets;
- using TAsyncUpdaterBase::SetUserContext;
-
- protected:
- NRoles::TRolesPtr Roles_;
- };
-}
diff --git a/library/cpp/tvmauth/client/ut/async_updater_ut.cpp b/library/cpp/tvmauth/client/ut/async_updater_ut.cpp
deleted file mode 100644
index 1c1e8cbaae..0000000000
--- a/library/cpp/tvmauth/client/ut/async_updater_ut.cpp
+++ /dev/null
@@ -1,165 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/misc/async_updater.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(AsyncUpdater) {
- static const TString SRV_TICKET = "3:serv:CBAQ__________9_IgYIexCUkQY:GioCM49Ob6_f80y6FY0XBVN4hLXuMlFeyMvIMiDuQnZkbkLpRpQOuQo5YjWoBjM0Vf-XqOm8B7xtrvxSYHDD7Q4OatN2l-Iwg7i71lE3scUeD36x47st3nd0OThvtjrFx_D8mw_c0GT5KcniZlqq1SjhLyAk1b_zJsx8viRAhCU";
- static const TString PROD_TICKET = "3:user:CAsQ__________9_Gg4KAgh7EHsg0oXYzAQoAA:N8PvrDNLh-5JywinxJntLeQGDEHBUxfzjuvB8-_BEUv1x9CALU7do8irDlDYVeVVDr4AIpR087YPZVzWPAqmnBuRJS0tJXekmDDvrivLnbRrzY4IUXZ_fImB0fJhTyVetKv6RD11bGqnAJeDpIukBwPTbJc_EMvKDt8V490CJFw";
- static const TString TEST_TICKET = "3:user:CA0Q__________9_Gg4KAgh7EHsg0oXYzAQoAQ:FSADps3wNGm92Vyb1E9IVq5M6ZygdGdt1vafWWEhfDDeCLoVA-sJesxMl2pGW4OxJ8J1r_MfpG3ZoBk8rLVMHUFrPa6HheTbeXFAWl8quEniauXvKQe4VyrpA1SPgtRoFqi5upSDIJzEAe1YRJjq1EClQ_slMt8R0kA_JjKUX54";
- static const TString PROD_YATEAM_TICKET = "3:user:CAwQ__________9_Gg4KAgh7EHsg0oXYzAQoAg:M9dEFEWHLHXiL7brCsyfYlm254PE6VeshUjI62u2qMDRzt6-0jAoJTIdDiogerItht1YFYSn8fSqmMf23_rueGj-wkmvyNzbcBSk3jtK2U5sai_W0bK6OwukR9tzWzi1Gcgg9DrNEeIKFvs1EBqYCF4mPHWo5bgk0CR580Cgit4";
- static const TString TEST_YATEAM_TICKET = "3:user:CA4Q__________9_Gg4KAgh7EHsg0oXYzAQoAw:IlaV3htk3jYrviIOz3k3Dfwz7p-bYYpbrgdn53GiUrMGdrT9eobHeuzNvPLrWB0yuYZAD46C3MGxok4GGmHhT73mki4XOCX8yWT4jW_hzcHBik1442tjWwh8IWqV_7q5j5496suVuLWjnZORWbb7I-2iwdIlU1BUiDfhoAolCq8";
- static const TString STRESS_TICKET = "3:user:CA8Q__________9_Gg4KAgh7EHsg0oXYzAQoBA:GBuG_TLo6SL2OYFxp7Zly04HPNzmAF7Fu2E8E9SnwQDoxq9rf7VThSPtTmnBSAl5UVRRPkMsRtzzHZ87qtj6l-PvF0K7PrDu7-yS_xiFTgAl9sEfXAIHJVzZLoksGRgpoBtpBUg9vVaJsPns0kWFKJgq8M-Mk9agrSk7sb2VUeQ";
-
- class TTestUpdater: public TAsyncUpdaterBase {
- public:
- using TAsyncUpdaterBase::SetBbEnv;
- using TAsyncUpdaterBase::SetUserContext;
-
- TClientStatus GetStatus() const override {
- return TClientStatus();
- }
- };
-
- Y_UNIT_TEST(User) {
- TTestUpdater u;
-
- UNIT_ASSERT(!u.GetCachedUserContext());
-
- u.SetUserContext(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT(!u.GetCachedUserContext());
-
- UNIT_ASSERT_NO_EXCEPTION(u.SetBbEnv(EBlackboxEnv::Prod));
- UNIT_ASSERT(u.GetCachedUserContext());
- UNIT_ASSERT(u.GetCachedUserContext()->Check(PROD_TICKET));
- UNIT_ASSERT_NO_EXCEPTION(u.GetCachedUserContext(EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(u.GetCachedUserContext(EBlackboxEnv::ProdYateam)->Check(PROD_YATEAM_TICKET));
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(u.SetBbEnv(EBlackboxEnv::Prod, EBlackboxEnv::Test),
- TBrokenTvmClientSettings,
- "Overriding of BlackboxEnv is illegal: Prod -> Test");
- UNIT_ASSERT_EXCEPTION_CONTAINS(u.GetCachedUserContext(EBlackboxEnv::Test),
- TBrokenTvmClientSettings,
- "Overriding of BlackboxEnv is illegal: Prod -> Test");
-
- UNIT_ASSERT(u.GetCachedUserContext());
- UNIT_ASSERT(u.GetCachedUserContext()->Check(PROD_TICKET));
- }
-
- class DummyUpdater: public TAsyncUpdaterBase {
- public:
- TClientStatus GetStatus() const override {
- return TClientStatus();
- }
-
- using TAsyncUpdaterBase::SetServiceContext;
- using TAsyncUpdaterBase::SetServiceTickets;
- using TAsyncUpdaterBase::SetUserContext;
- };
-
- Y_UNIT_TEST(Cache) {
- DummyUpdater d;
-
- UNIT_ASSERT(!d.GetCachedServiceTickets());
- TServiceTicketsPtr st = MakeIntrusiveConst<TServiceTickets>(TServiceTickets::TMapIdStr(),
- TServiceTickets::TMapIdStr(),
- TServiceTickets::TMapAliasId());
- d.SetServiceTickets(st);
- UNIT_ASSERT_EQUAL(st.Get(), d.GetCachedServiceTickets().Get());
-
- UNIT_ASSERT(!d.GetCachedServiceContext());
- TServiceContextPtr sc = MakeIntrusiveConst<TServiceContext>(TServiceContext::SigningFactory("kjndfadfndsfafdasd"));
- d.SetServiceContext(sc);
- UNIT_ASSERT_EQUAL(sc.Get(), d.GetCachedServiceContext().Get());
-
- UNIT_ASSERT(!d.GetCachedUserContext());
- d.SetUserContext(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- }
-
- Y_UNIT_TEST(ServiceTickets_Aliases) {
- using TId = TServiceTickets::TMapIdStr;
- using TUnfetchedId = TServiceTickets::TIdSet;
- using TStr = TServiceTickets::TMapAliasStr;
- using TUnfetchedAlias = TServiceTickets::TAliasSet;
- using TAls = TServiceTickets::TMapAliasId;
- TServiceTickets t(TId{}, TId{}, TAls{});
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({{1, "t1"}, {2, "t2"}}),
- TId({{3, "e1"}}),
- TAls()));
- UNIT_ASSERT_EQUAL(TId({{1, "t1"}, {2, "t2"}}), t.TicketsById);
- UNIT_ASSERT_EQUAL(TId({{3, "e1"}}), t.ErrorsById);
- UNIT_ASSERT_EQUAL(TStr(), t.TicketsByAlias);
- UNIT_ASSERT_EQUAL(TStr(), t.ErrorsByAlias);
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({{1, "t1"}, {2, "t2"}}),
- TId({{3, "e1"}}),
- TAls({{"1", 1}, {"2", 2}, {"3", 3}})));
- UNIT_ASSERT_EQUAL(TId({{1, "t1"}, {2, "t2"}}), t.TicketsById);
- UNIT_ASSERT_EQUAL(TId({{3, "e1"}}), t.ErrorsById);
- UNIT_ASSERT_EQUAL(TUnfetchedId(), t.UnfetchedIds);
- UNIT_ASSERT_EQUAL(TStr({{"1", "t1"}, {"2", "t2"}}), t.TicketsByAlias);
- UNIT_ASSERT_EQUAL(TStr({{"3", "e1"}}), t.ErrorsByAlias);
- UNIT_ASSERT_EQUAL(TUnfetchedAlias({}), t.UnfetchedAliases);
- }
-
- Y_UNIT_TEST(ServiceTickets_UnfetchedIds) {
- using TId = TServiceTickets::TMapIdStr;
- using TUnfetchedId = TServiceTickets::TIdSet;
- using TStr = TServiceTickets::TMapAliasStr;
- using TUnfetchedAlias = TServiceTickets::TAliasSet;
- using TAls = TServiceTickets::TMapAliasId;
- TServiceTickets t(TId({{1, "t1"}, {2, "t2"}}),
- TId(),
- TAls({{"1", 1}, {"2", 2}, {"3", 3}}));
-
- UNIT_ASSERT_EQUAL(TId({{1, "t1"}, {2, "t2"}}), t.TicketsById);
- UNIT_ASSERT_EQUAL(TId({}), t.ErrorsById);
- UNIT_ASSERT_EQUAL(TUnfetchedId({3}), t.UnfetchedIds);
- UNIT_ASSERT_EQUAL(TUnfetchedAlias({{"3"}}), t.UnfetchedAliases);
- UNIT_ASSERT_EQUAL(TStr({{"1", "t1"}, {"2", "t2"}}), t.TicketsByAlias);
- UNIT_ASSERT_EQUAL(TStr(), t.ErrorsByAlias);
- }
-
- Y_UNIT_TEST(ServiceTickets_InvalidationTime) {
- using TId = TServiceTickets::TMapIdStr;
- using TAls = TServiceTickets::TMapAliasId;
-
- TServiceTickets t(TId{}, TId{}, TAls{});
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), t.InvalidationTime);
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({{1, SRV_TICKET}}),
- TId(),
- TAls()));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(std::numeric_limits<time_t>::max()), t.InvalidationTime);
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({
- {1, SRV_TICKET},
- {2, "serv"},
- }),
- TId(),
- TAls()));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(std::numeric_limits<time_t>::max()), t.InvalidationTime);
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({
- {2, "serv"},
- }),
- TId(),
- TAls()));
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), t.InvalidationTime);
-
- UNIT_ASSERT_NO_EXCEPTION(t = TServiceTickets(TId({
- {1, SRV_TICKET},
- {2, "serv"},
- {3, "3:serv:CBAQeyIECAMQAw:TiZjG2Ut9j-9n0zcqxGW8xiYmnFa-i10-dbA0FKIInKzeDuueovWVEBcgbQHndblzRCxoIBMgbotOf7ALk2xoSBnRbOKomAIEtiTBL77GByL5O8K_HUGNYb-ygqnmZlIuLalgeRQAdsKstgUwQzufnOQyekipmamwo7EVQhr8Ug"},
- }),
- TId(),
- TAls()));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(123), t.InvalidationTime);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/checker_ut.cpp b/library/cpp/tvmauth/client/ut/checker_ut.cpp
deleted file mode 100644
index 54a25974c1..0000000000
--- a/library/cpp/tvmauth/client/ut/checker_ut.cpp
+++ /dev/null
@@ -1,176 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/mocked_updater.h>
-#include <library/cpp/tvmauth/client/misc/checker.h>
-#include <library/cpp/tvmauth/client/misc/getter.h>
-#include <library/cpp/tvmauth/client/misc/api/threaded_updater.h>
-
-#include <library/cpp/tvmauth/type.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientChecker) {
- static const TTvmId OK_CLIENT = 100500;
- static const TString PROD_TICKET = "3:user:CAsQ__________9_Gg4KAgh7EHsg0oXYzAQoAA:N8PvrDNLh-5JywinxJntLeQGDEHBUxfzjuvB8-_BEUv1x9CALU7do8irDlDYVeVVDr4AIpR087YPZVzWPAqmnBuRJS0tJXekmDDvrivLnbRrzY4IUXZ_fImB0fJhTyVetKv6RD11bGqnAJeDpIukBwPTbJc_EMvKDt8V490CJFw";
- static const TString TEST_TICKET = "3:user:CA0Q__________9_Gg4KAgh7EHsg0oXYzAQoAQ:FSADps3wNGm92Vyb1E9IVq5M6ZygdGdt1vafWWEhfDDeCLoVA-sJesxMl2pGW4OxJ8J1r_MfpG3ZoBk8rLVMHUFrPa6HheTbeXFAWl8quEniauXvKQe4VyrpA1SPgtRoFqi5upSDIJzEAe1YRJjq1EClQ_slMt8R0kA_JjKUX54";
- static const TString PROD_YATEAM_TICKET = "3:user:CAwQ__________9_Gg4KAgh7EHsg0oXYzAQoAg:M9dEFEWHLHXiL7brCsyfYlm254PE6VeshUjI62u2qMDRzt6-0jAoJTIdDiogerItht1YFYSn8fSqmMf23_rueGj-wkmvyNzbcBSk3jtK2U5sai_W0bK6OwukR9tzWzi1Gcgg9DrNEeIKFvs1EBqYCF4mPHWo5bgk0CR580Cgit4";
- static const TString TEST_YATEAM_TICKET = "3:user:CA4Q__________9_Gg4KAgh7EHsg0oXYzAQoAw:IlaV3htk3jYrviIOz3k3Dfwz7p-bYYpbrgdn53GiUrMGdrT9eobHeuzNvPLrWB0yuYZAD46C3MGxok4GGmHhT73mki4XOCX8yWT4jW_hzcHBik1442tjWwh8IWqV_7q5j5496suVuLWjnZORWbb7I-2iwdIlU1BUiDfhoAolCq8";
- static const TString STRESS_TICKET = "3:user:CA8Q__________9_Gg4KAgh7EHsg0oXYzAQoBA:GBuG_TLo6SL2OYFxp7Zly04HPNzmAF7Fu2E8E9SnwQDoxq9rf7VThSPtTmnBSAl5UVRRPkMsRtzzHZ87qtj6l-PvF0K7PrDu7-yS_xiFTgAl9sEfXAIHJVzZLoksGRgpoBtpBUg9vVaJsPns0kWFKJgq8M-Mk9agrSk7sb2VUeQ";
- static const TString SRV_TICKET = "3:serv:CBAQ__________9_IgYIexCUkQY:GioCM49Ob6_f80y6FY0XBVN4hLXuMlFeyMvIMiDuQnZkbkLpRpQOuQo5YjWoBjM0Vf-XqOm8B7xtrvxSYHDD7Q4OatN2l-Iwg7i71lE3scUeD36x47st3nd0OThvtjrFx_D8mw_c0GT5KcniZlqq1SjhLyAk1b_zJsx8viRAhCU";
-
- Y_UNIT_TEST(User) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(OK_CLIENT);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(GetCachePath());
-
- auto l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- UNIT_ASSERT_EXCEPTION(TUserTicketChecker(u), TBrokenTvmClientSettings);
- }
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully fetched") == TString::npos, l->Stream.Str());
-
- s.EnableUserTicketChecking(EBlackboxEnv::Prod);
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TUserTicketChecker c(u);
- UNIT_ASSERT(c.Check(PROD_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_TICKET, {}));
- UNIT_ASSERT(!c.Check(PROD_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(STRESS_TICKET, {}));
-
- UNIT_ASSERT(!c.Check(PROD_TICKET, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!c.Check(TEST_TICKET, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(c.Check(PROD_YATEAM_TICKET, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!c.Check(TEST_YATEAM_TICKET, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!c.Check(STRESS_TICKET, EBlackboxEnv::ProdYateam));
-
- UNIT_ASSERT_EXCEPTION(c.Check(PROD_TICKET, EBlackboxEnv::Stress), TBrokenTvmClientSettings);
- }
-
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TUserTicketChecker c(u);
- UNIT_ASSERT(!c.Check(PROD_TICKET, {}));
- UNIT_ASSERT(c.Check(TEST_TICKET, {}));
- UNIT_ASSERT(!c.Check(PROD_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(STRESS_TICKET, {}));
- }
-
- s.EnableUserTicketChecking(EBlackboxEnv::ProdYateam);
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TUserTicketChecker c(u);
- UNIT_ASSERT(!c.Check(PROD_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_TICKET, {}));
- UNIT_ASSERT(c.Check(PROD_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(STRESS_TICKET, {}));
- }
-
- s.EnableUserTicketChecking(EBlackboxEnv::TestYateam);
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TUserTicketChecker c(u);
- UNIT_ASSERT(!c.Check(PROD_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_TICKET, {}));
- UNIT_ASSERT(!c.Check(PROD_YATEAM_TICKET, {}));
- UNIT_ASSERT(c.Check(TEST_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(STRESS_TICKET, {}));
- }
-
- s.EnableUserTicketChecking(EBlackboxEnv::Stress);
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TUserTicketChecker c(u);
- UNIT_ASSERT(c.Check(PROD_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_TICKET, {}));
- UNIT_ASSERT(!c.Check(PROD_YATEAM_TICKET, {}));
- UNIT_ASSERT(!c.Check(TEST_YATEAM_TICKET, {}));
- UNIT_ASSERT(c.Check(STRESS_TICKET, {}));
- }
- }
-
- Y_UNIT_TEST(Service) {
- NTvmApi::TClientSettings s;
- s.EnableUserTicketChecking(EBlackboxEnv::Stress);
- s.SetSelfTvmId(OK_CLIENT);
- s.SetDiskCacheDir(GetCachePath());
-
- auto l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- UNIT_ASSERT_EXCEPTION(TServiceTicketChecker(u), TBrokenTvmClientSettings);
- }
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully fetched") == TString::npos, l->Stream.Str());
-
- s.EnableServiceTicketChecking();
- l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TServiceTicketChecker c(u);
- UNIT_ASSERT(c.Check(SRV_TICKET));
- UNIT_ASSERT(!c.Check(PROD_TICKET));
- }
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully fetched") == TString::npos, l->Stream.Str());
-
- s.SetSelfTvmId(17);
- l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TServiceTicketChecker c(u);
- UNIT_ASSERT(!c.Check(SRV_TICKET));
- UNIT_ASSERT(!c.Check(PROD_TICKET));
- }
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully fetched") == TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(Tickets) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(OK_CLIENT);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(GetCachePath());
-
- auto l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus().GetCode());
- TServiceTicketGetter g(u);
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIKhCUkQY:CX", g.GetTicket("blackbox"));
- UNIT_ASSERT_EXCEPTION_CONTAINS(g.GetTicket("blackbox2"),
- TBrokenTvmClientSettings,
- "Destination 'blackbox2' was not specified in settings. Check your settings (if you use Qloud/YP/tvmtool - check it's settings)");
- }
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully fetched") == TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(ErrorForDst) {
- TServiceTicketGetter g(new TMockedUpdater);
-
- UNIT_ASSERT_VALUES_EQUAL(TMockedUpdater::TSettings::CreateDeafult().Backends.at(0).Value,
- g.GetTicket("my_dest"));
- UNIT_ASSERT_VALUES_EQUAL(TMockedUpdater::TSettings::CreateDeafult().Backends.at(0).Value,
- g.GetTicket(42));
- UNIT_ASSERT_EXCEPTION_CONTAINS(g.GetTicket("my_bad_dest"),
- TMissingServiceTicket,
- "Failed to get ticket for 'my_bad_dest': Dst is not found");
- UNIT_ASSERT_EXCEPTION_CONTAINS(g.GetTicket(43),
- TMissingServiceTicket,
- "Failed to get ticket for '43': Dst is not found");
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/client_status_ut.cpp b/library/cpp/tvmauth/client/ut/client_status_ut.cpp
deleted file mode 100644
index a1c3ae74ce..0000000000
--- a/library/cpp/tvmauth/client/ut/client_status_ut.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include <library/cpp/tvmauth/client/client_status.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientStatus) {
- Y_UNIT_TEST(Common) {
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, TClientStatus().GetCode());
- UNIT_ASSERT_VALUES_EQUAL("", TClientStatus().GetLastError());
-
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, TClientStatus(TClientStatus::Warning, "kek"));
- UNIT_ASSERT_VALUES_EQUAL("kek",
- TClientStatus(TClientStatus::Warning, "kek").GetLastError());
- UNIT_ASSERT_VALUES_EQUAL("2;TvmClient: kek\n",
- TClientStatus(TClientStatus::Error, "kek").CreateJugglerMessage());
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/common.h b/library/cpp/tvmauth/client/ut/common.h
deleted file mode 100644
index 0aee09aefc..0000000000
--- a/library/cpp/tvmauth/client/ut/common.h
+++ /dev/null
@@ -1,240 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/client/logger.h>
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-#include <library/cpp/tvmauth/client/misc/roles/entities_index.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/cgiparam/cgiparam.h>
-#include <library/cpp/testing/mock_server/server.h>
-#include <library/cpp/testing/unittest/env.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/stream/str.h>
-#include <util/system/fs.h>
-
-class TLogger: public NTvmAuth::ILogger {
-public:
- void Log(int lvl, const TString& msg) override {
- Cout << TInstant::Now() << " lvl=" << lvl << " msg: " << msg << "\n";
- Stream << lvl << ": " << msg << Endl;
- }
-
- TStringStream Stream;
-};
-
-static inline TString GetFilePath(const char* name) {
- return ArcadiaSourceRoot() + "/library/cpp/tvmauth/client/ut/files/" + name;
-}
-
-static inline TString GetCachePath(const TString& dir = {}) {
- if (dir) {
- Y_ENSURE(NFs::MakeDirectoryRecursive("./" + dir));
- }
-
- auto wr = [](const TString& p, const TStringBuf body) {
- NTvmAuth::TDiskWriter w(p);
- Y_ENSURE(w.Write(body, TInstant::ParseIso8601("2050-01-01T00:00:00.000000Z")));
- };
- wr("./" + dir + "/public_keys", NTvmAuth::NUnittest::TVMKNIFE_PUBLIC_KEYS);
- wr("./" + dir + "/service_tickets",
- R"({
- "19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- "213" : { "ticket" : "service_ticket_2"},
- "234" : { "error" : "Dst is not found" },
- "185" : { "ticket" : "service_ticket_3"}
-} 100500)");
-
- return "./" + dir;
-}
-
-static const TString AUTH_TOKEN = "strong_token";
-static const TString META = R"(
-{
-"bb_env" : "ProdYaTeam",
-"tenants" : [
- {
- "self": {
- "alias" : "me",
- "client_id": 100500
- },
- "dsts" : [
- {
- "alias" : "bbox",
- "client_id": 242
- },
- {
- "alias" : "pass_likers",
- "client_id": 11
- }
- ]
- },
- {
- "self": {
- "alias" : "tenant_with_roles",
- "client_id": 100500
- },
- "idm_slug": "some_slug",
- "dsts" : []
- },
- {
- "self": {
- "alias" : "push-client",
- "client_id": 100501
- },
- "dsts" : [
- {
- "alias" : "pass_likers",
- "client_id": 100502
- }
- ]
- },
- {
- "self": {
- "alias" : "multi_names_for_dst",
- "client_id": 100599
- },
- "dsts" : [
- {
- "alias" : "pass_likers",
- "client_id": 100502
- },
- {
- "alias" : "pass_haters",
- "client_id": 100502
- }
- ]
- },
- {
- "self": {
- "alias" : "something_else",
- "client_id": 100503
- },
- "dsts" : [
- ]
- }
-]
-})";
-
-static const TString TICKETS_ME =
- R"({
- "pass_likers": {
- "ticket": "3:serv:CBAQ__________9_IgYIlJEGEAs:T-apeMNWFc_vHPQ3iLaZv9NjG-hf5-i23O4AhRu1M68ryN3FU5qvyqTSSiPbtJdFP6EE41QQBzEs59dHn9DRkqQNwwKf1is00Oewwj2XKO0uHukuzd9XxZnro7MfjPswsjWufxX28rmJtlfSXwAtyKt8TI5yKJnMeBPQ0m5R3k8",
- "tvm_id": 11
- },
- "bbox": {
- "ticket": "3:serv:CBAQ__________9_IgcIlJEGEPIB:N7luw0_rVmBosTTI130jwDbQd0-cMmqJeEl0ma4ZlIo_mHXjBzpOuMQ3A9YagbmOBOt8TZ_gzGvVSegWZkEeB24gM22acw0w-RcHaQKrzSOA5Zq8WLNIC8QUa4_WGTlAsb7R7eC4KTAGgouIquNAgMBdTuGOuZHnMLvZyLnOMKc",
- "tvm_id": 242
- }
- })";
-
-static const TString SERVICE_TICKET_PC = "3:serv:CBAQ__________9_IggIlpEGEJaRBg:BAxaQJCdK4eFuJ6i_egqPwvJgWtlh0enDQRPr84Nx2phZ_8QtxKAUCwEa7KOU_jVvIBQIC5-ETTl2vjBt7UyygF8frdK4ab6zJoWj4n07np6vbmWd385l8KvzztLt4QkBrPiE7U46dK3pL0U8tfBkSXE8rvUIsl3RvvgSNH2J3c";
-static const TString TICKETS_PC =
- R"({
- "pass_likers": {
- "ticket": "3:serv:CBAQ__________9_IggIlpEGEJaRBg:BAxaQJCdK4eFuJ6i_egqPwvJgWtlh0enDQRPr84Nx2phZ_8QtxKAUCwEa7KOU_jVvIBQIC5-ETTl2vjBt7UyygF8frdK4ab6zJoWj4n07np6vbmWd385l8KvzztLt4QkBrPiE7U46dK3pL0U8tfBkSXE8rvUIsl3RvvgSNH2J3c",
- "tvm_id": 100502
- }
- })";
-
-static const TString TICKETS_MANY_DSTS =
- R"({
- "pass_likers": {
- "ticket": "3:serv:CBAQ__________9_IggI95EGEJaRBg:D0MOLDhKQyI-OhC0ON9gYukz2hOctUipu1yXsvkw6NRuLhcBfvGayyUqF4ILrqepjz9GtPWIR_wO6oLSW35Z0YaFn60QWp5tG6IcAnr80lm_OnLHJt4kmEoLtGg1V0aWBT0YyouzGB2-QFNOVO86G7sYzU8FC6-V3Iyc4X7XTNc",
- "tvm_id": 100502
- },
- "who_are_you??": {
- "ticket": "kek",
- "tvm_id": 100503
- },
- "pass_haters": {
- "ticket": "3:serv:CBAQ__________9_IggI95EGEJaRBg:D0MOLDhKQyI-OhC0ON9gYukz2hOctUipu1yXsvkw6NRuLhcBfvGayyUqF4ILrqepjz9GtPWIR_wO6oLSW35Z0YaFn60QWp5tG6IcAnr80lm_OnLHJt4kmEoLtGg1V0aWBT0YyouzGB2-QFNOVO86G7sYzU8FC6-V3Iyc4X7XTNc",
- "tvm_id": 100502
- }
- })";
-
-static const TString TICKETS_SE = R"({})";
-
-static const TInstant BIRTHTIME = TInstant::Seconds(14380887840);
-class TTvmTool: public TRequestReplier {
-public:
- TString Meta;
- HttpCodes Code;
- TInstant Birthtime;
-
- TTvmTool()
- : Meta(META)
- , Code(HTTP_OK)
- , Birthtime(BIRTHTIME)
- {
- }
-
- bool DoReply(const TReplyParams& params) override {
- const TParsedHttpFull http(params.Input.FirstLine());
- if (http.Path == "/tvm/ping") {
- THttpResponse resp(HTTP_OK);
- resp.SetContent("OK");
- resp.OutTo(params.Output);
- return true;
- }
-
- auto it = std::find_if(params.Input.Headers().begin(),
- params.Input.Headers().end(),
- [](const THttpInputHeader& h) { return h.Name() == "Authorization"; });
- if (it == params.Input.Headers().end() || it->Value() != AUTH_TOKEN) {
- THttpResponse resp(HTTP_UNAUTHORIZED);
- resp.SetContent("pong");
- resp.OutTo(params.Output);
- return true;
- }
-
- THttpResponse resp(Code);
- if (http.Path == "/tvm/keys") {
- resp.SetContent(NTvmAuth::NUnittest::TVMKNIFE_PUBLIC_KEYS);
- } else if (http.Path == "/tvm/tickets") {
- TCgiParameters cg;
- cg.ScanAddAll(http.Cgi);
- if (cg.Get("src") == "100500") {
- resp.SetContent(TICKETS_ME);
- } else if (cg.Get("src") == "100501") {
- resp.SetContent(TICKETS_PC);
- } else if (cg.Get("src") == "100599") {
- resp.SetContent(TICKETS_MANY_DSTS);
- }
- } else if (http.Path == "/tvm/private_api/__meta__") {
- resp.SetContent(Meta);
- }
- resp.AddHeader("X-Ya-Tvmtool-Data-Birthtime", IntToString<10>(Birthtime.Seconds()));
- resp.OutTo(params.Output);
-
- return true;
- }
-};
-
-static inline NTvmAuth::NRoles::TEntitiesIndex CreateEntitiesIndex() {
- using namespace NTvmAuth::NRoles;
-
- TEntitiesIndex index(
- {
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- }),
- });
-
- return index;
-}
diff --git a/library/cpp/tvmauth/client/ut/default_uid_checker_ut.cpp b/library/cpp/tvmauth/client/ut/default_uid_checker_ut.cpp
deleted file mode 100644
index a92530cab1..0000000000
--- a/library/cpp/tvmauth/client/ut/default_uid_checker_ut.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/mocked_updater.h>
-#include <library/cpp/tvmauth/client/misc/default_uid_checker.h>
-#include <library/cpp/tvmauth/client/misc/api/threaded_updater.h>
-
-#include <library/cpp/tvmauth/type.h>
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(DefaultUidChecker) {
- Y_UNIT_TEST(Ctor) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDefaultUidChecker(new TMockedUpdater),
- TBrokenTvmClientSettings,
- "Need to use TClientSettings::EnableRolesFetching");
- }
-
- Y_UNIT_TEST(Check) {
- NRoles::TRolesPtr roles = std::make_shared<NRoles::TRoles>(
- NRoles::TRoles::TMeta{},
- NRoles::TRoles::TTvmConsumers{},
- NRoles::TRoles::TUserConsumers{
- {12345, std::make_shared<NRoles::TConsumerRoles>(
- THashMap<TString, NRoles::TEntitiesPtr>())},
- },
- std::make_shared<TString>());
- const TDefaultUidChecker checker(new TMockedUpdater({.Roles = roles}));
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- checker.Check(NUnittest::CreateUserTicket(ETicketStatus::Expired, 12345, {})),
- TIllegalUsage,
- "User ticket must be valid");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- checker.Check(NUnittest::CreateUserTicket(ETicketStatus::Ok, 12345, {}, {}, EBlackboxEnv::Test)),
- TIllegalUsage,
- "User ticket must be from ProdYateam, got from Test");
-
- TCheckedUserTicket ticket;
- UNIT_ASSERT_NO_EXCEPTION(
- ticket = checker.Check(NUnittest::CreateUserTicket(ETicketStatus::Ok, 12345, {}, {}, EBlackboxEnv::ProdYateam)));
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, ticket.GetStatus());
-
- UNIT_ASSERT_NO_EXCEPTION(
- ticket = checker.Check(NUnittest::CreateUserTicket(ETicketStatus::Ok, 9999, {}, {}, EBlackboxEnv::ProdYateam)));
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::NoRoles, ticket.GetStatus());
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/disk_cache_ut.cpp b/library/cpp/tvmauth/client/ut/disk_cache_ut.cpp
deleted file mode 100644
index 7dd851c9b3..0000000000
--- a/library/cpp/tvmauth/client/ut/disk_cache_ut.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/logger.h>
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/stream/file.h>
-#include <util/system/fs.h>
-#include <util/system/sysstat.h>
-
-#include <thread>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientDisk) {
- Y_UNIT_TEST(Hash) {
- TString hash = TDiskReader::GetHash("asd");
- UNIT_ASSERT(hash);
- UNIT_ASSERT_VALUES_EQUAL(32, hash.size());
- UNIT_ASSERT_VALUES_EQUAL("Zj5_qYg31bPlqjBW76z8IV0rCsHmv-iN-McV6ybS1-g", NUtils::Bin2base64url(hash));
- }
-
- Y_UNIT_TEST(Timestamp) {
- time_t t = 100500;
-
- TString s = TDiskWriter::WriteTimestamp(t);
- UNIT_ASSERT_VALUES_EQUAL("lIgBAAAAAAA", NUtils::Bin2base64url(s));
- UNIT_ASSERT_VALUES_EQUAL(t, TDiskReader::GetTimestamp(s));
-
- t = 123123123213089;
- s = TDiskWriter::WriteTimestamp(t);
- UNIT_ASSERT_VALUES_EQUAL("IdMF1vpvAAA", NUtils::Bin2base64url(s));
- UNIT_ASSERT_VALUES_EQUAL(t, TDiskReader::GetTimestamp(s));
-
- t = time(nullptr);
- s = TDiskWriter::WriteTimestamp(t);
- UNIT_ASSERT_VALUES_EQUAL(t, TDiskReader::GetTimestamp(s));
- }
-
- const TInstant TIME = TInstant::Seconds(100500);
- const TString DATA = "oiweuhn \n vw3ut hweoi uhgewproritjhwequtherwoiughfdsv 8ty34q01u 34 1=3";
-
- Y_UNIT_TEST(ParseData_Ok) {
- TLogger l;
-
- const TInstant time = TInstant::Seconds(1523446554789);
-
- TString toFile = TDiskWriter::PrepareData(time, DATA);
- UNIT_ASSERT_VALUES_EQUAL(113, toFile.size());
- UNIT_ASSERT_VALUES_EQUAL("T8BnRIMoC6mlMXexPg9cV5jYxeFtgDWk97JTajHDunCloH20YgEAAG9pd2V1aG4gCiB2dzN1dCBod2VvaSB1aGdld3Byb3JpdGpod2VxdXRoZXJ3b2l1Z2hmZHN2IDh0eTM0cTAxdSAgIDM0ICAxPTM",
- NUtils::Bin2base64url(toFile));
-
- TDiskReader r("qwerty", &l);
- UNIT_ASSERT(r.ParseData(toFile));
- UNIT_ASSERT_VALUES_EQUAL(DATA, r.Data());
- UNIT_ASSERT_VALUES_EQUAL(time, r.Time());
- UNIT_ASSERT_VALUES_EQUAL("6: File 'qwerty' was successfully read\n",
- l.Stream.Str());
- }
-
- Y_UNIT_TEST(ParseData_SmallFile) {
- TLogger l;
-
- TString toFile = TDiskWriter::PrepareData(TIME, DATA);
- TDiskReader r("qwerty", &l);
- UNIT_ASSERT(!r.ParseData(toFile.substr(0, 17)));
- UNIT_ASSERT_VALUES_EQUAL("4: File 'qwerty' is too small\n",
- l.Stream.Str());
- }
-
- Y_UNIT_TEST(ParseData_Changed) {
- TLogger l;
-
- TString toFile = TDiskWriter::PrepareData(TIME, DATA);
- toFile[17] = toFile[17] + 1;
- TDiskReader r("qwerty", &l);
- UNIT_ASSERT(!r.ParseData(toFile));
- UNIT_ASSERT_VALUES_EQUAL("4: Content of 'qwerty' was incorrectly changed\n",
- l.Stream.Str());
- }
-
- Y_UNIT_TEST(Read_Ok) {
- TLogger l;
-
- TDiskReader r(GetFilePath("ok.cache"), &l);
- UNIT_ASSERT(r.Read());
- UNIT_ASSERT_VALUES_EQUAL(DATA, r.Data());
- UNIT_ASSERT_VALUES_EQUAL(TIME, r.Time());
- UNIT_ASSERT_C(l.Stream.Str().find("was successfully read") != TString::npos, l.Stream.Str());
- }
-
- Y_UNIT_TEST(Read_NoFile) {
- TLogger l;
-
- TDiskReader r("missing", &l);
- UNIT_ASSERT(!r.Read());
- UNIT_ASSERT_VALUES_EQUAL("7: File 'missing' does not exist\n",
- l.Stream.Str());
- }
-
-#ifdef _unix_
- Y_UNIT_TEST(Read_NoPermitions) {
- TLogger l;
-
- const TString path = GetWorkPath() + "/123";
- {
- TFileOutput output(path);
- }
- Chmod(path.data(), S_IWUSR);
-
- TDiskReader r(path, &l);
- UNIT_ASSERT(!r.Read());
- UNIT_ASSERT_C(l.Stream.Str().find("Permission denied") != TString::npos, l.Stream.Str());
-
- Chmod(path.data(), S_IRWXU);
- NFs::Remove(path);
- }
-#endif
-
- Y_UNIT_TEST(Write_Ok) {
- TLogger l;
-
- const TString path = "./tmp_file";
- TDiskWriter w(path, &l);
- UNIT_ASSERT_C(w.Write(DATA), l.Stream.Str());
- UNIT_ASSERT_C(l.Stream.Str().find("was successfully written") != TString::npos, l.Stream.Str());
- l.Stream.Clear();
-
- TDiskReader r(path, &l);
- UNIT_ASSERT_C(r.Read(), l.Stream.Str());
- UNIT_ASSERT_VALUES_EQUAL(DATA, r.Data());
- UNIT_ASSERT(TInstant::Now() - r.Time() < TDuration::Minutes(5));
- UNIT_ASSERT_C(l.Stream.Str().find("was successfully read") != TString::npos, l.Stream.Str());
-
- NFs::Remove(path);
- }
-
- Y_UNIT_TEST(Write_NoPermitions) {
- TLogger l;
-
- TDiskWriter w("/some_file", &l);
- UNIT_ASSERT(!w.Write(DATA));
- UNIT_ASSERT_C(l.Stream.Str().Contains("3: Failed to write '/some_file': ("), l.Stream.Str());
- UNIT_ASSERT_C(l.Stream.Str().Contains("denied"), l.Stream.Str());
- }
-
- Y_UNIT_TEST(race) {
- const TString path = "./tmp_file";
- const TString data = "ejufhsadkjfvbhsaoicnaofssdahfasdfhasdofdsaf";
- NFs::Remove(path);
-
- std::atomic<bool> fail = false;
- std::vector<std::thread> thrs;
- for (size_t idx = 0; idx < 16; ++idx) {
- thrs.push_back(std::thread([&fail, data, path]() {
- TDiskWriter w(path);
- for (size_t k = 0; k < 1000; ++k) {
- if (!w.Write(data)) {
- fail = true;
- }
- }
- }));
- }
- for (std::thread& t : thrs) {
- t.join();
- }
- thrs.clear();
- UNIT_ASSERT(fail);
- {
- TDiskWriter w(path);
- UNIT_ASSERT(w.Write(data)); // checks unlocked flock
- }
-
- fail = false;
-
- for (size_t idx = 0; idx < 4; ++idx) {
- thrs.push_back(std::thread([&fail, data, path]() {
- TLogger l;
- TDiskReader r(path, &l);
- for (size_t k = 0; k < 100; ++k) {
- if (!r.Read()) {
- Cerr << l.Stream.Str() << Flush;
- fail = true;
- return;
- }
- if (r.Data() != data) {
- Cerr << (TStringBuilder() << "'" << data << "' vs '" << r.Data() << "'" << Endl) << Flush;
- fail = true;
- return;
- }
- }
- }));
- }
- for (std::thread& t : thrs) {
- t.join();
- }
- thrs.clear();
- UNIT_ASSERT(!fail);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/exponential_backoff_ut.cpp b/library/cpp/tvmauth/client/ut/exponential_backoff_ut.cpp
deleted file mode 100644
index 3dcbe6ad49..0000000000
--- a/library/cpp/tvmauth/client/ut/exponential_backoff_ut.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-#include <library/cpp/tvmauth/client/misc/exponential_backoff.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <thread>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(PasspUtilsExpBackoff) {
- Y_UNIT_TEST(common) {
- TExponentialBackoff b({TDuration::Seconds(1), TDuration::Seconds(60), 2, 0.01});
-
- UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(1), b.GetCurrentValue());
-
- TDuration dur = b.GetCurrentValue();
- for (size_t idx = 0; idx < 6; ++idx) {
- TDuration newValue = b.Increase();
- UNIT_ASSERT_LT(dur, newValue);
- dur = newValue;
- }
-
- UNIT_ASSERT_LT(TDuration::Seconds(60) - TDuration::Seconds(3), dur);
- UNIT_ASSERT_LT(dur, TDuration::Seconds(60) + TDuration::Seconds(3));
- }
-
- Y_UNIT_TEST(sleep) {
- TExponentialBackoff b({TDuration::Seconds(60), TDuration::Seconds(600), 2, 0.01});
-
- const TInstant start = TInstant::Now();
-
- TAutoEvent started;
- std::thread t([&b, &started]() {
- started.Signal();
- b.Sleep();
- });
-
- started.WaitT(TDuration::Seconds(30));
- b.Interrupt();
- t.join();
- TDuration dur = TInstant::Now() - start;
-
- UNIT_ASSERT_LT(dur, TDuration::Seconds(60));
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/facade_ut.cpp b/library/cpp/tvmauth/client/ut/facade_ut.cpp
deleted file mode 100644
index 0244521ef4..0000000000
--- a/library/cpp/tvmauth/client/ut/facade_ut.cpp
+++ /dev/null
@@ -1,167 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/facade.h>
-#include <library/cpp/tvmauth/client/mocked_updater.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/vector.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientFacade) {
- static const TTvmId OK_CLIENT = 100500;
- static const TString SRV_TICKET_123 = "3:serv:CBAQ__________9_IgYIexCUkQY:GioCM49Ob6_f80y6FY0XBVN4hLXuMlFeyMvIMiDuQnZkbkLpRpQOuQo5YjWoBjM0Vf-XqOm8B7xtrvxSYHDD7Q4OatN2l-Iwg7i71lE3scUeD36x47st3nd0OThvtjrFx_D8mw_c0GT5KcniZlqq1SjhLyAk1b_zJsx8viRAhCU";
- static const TString SRV_TICKET_456 = "3:serv:CBAQ__________9_IgcIyAMQlJEG:VrnqRhpoiDnJeAQbySJluJ1moQ5Kemic99iWzOrHLGfuh7iTw_xMT7KewRAmZMUwDKzE6otj7V86Xsnxbv5xZl8746wbvNcyUXu-nGWmbByZjO7xpSIcY07sISqEhP9n9C_yMSvqDP7ho_PRIfpGCDMXxKlFZ_BhBLLp0kHEvw4";
- static const TString PROD_TICKET = "3:user:CAsQ__________9_Gg4KAgh7EHsg0oXYzAQoAA:N8PvrDNLh-5JywinxJntLeQGDEHBUxfzjuvB8-_BEUv1x9CALU7do8irDlDYVeVVDr4AIpR087YPZVzWPAqmnBuRJS0tJXekmDDvrivLnbRrzY4IUXZ_fImB0fJhTyVetKv6RD11bGqnAJeDpIukBwPTbJc_EMvKDt8V490CJFw";
- static const TString TEST_TICKET = "3:user:CA0Q__________9_Gg4KAgh7EHsg0oXYzAQoAQ:FSADps3wNGm92Vyb1E9IVq5M6ZygdGdt1vafWWEhfDDeCLoVA-sJesxMl2pGW4OxJ8J1r_MfpG3ZoBk8rLVMHUFrPa6HheTbeXFAWl8quEniauXvKQe4VyrpA1SPgtRoFqi5upSDIJzEAe1YRJjq1EClQ_slMt8R0kA_JjKUX54";
-
- TTvmClient GetClient(const NTvmApi::TClientSettings& s) {
- auto l = MakeIntrusive<TLogger>();
- TTvmClient f(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, f.GetStatus());
- Sleep(TDuration::MilliSeconds(300));
- TString logs = l->Stream.Str();
- UNIT_ASSERT_C(logs.find("was successfully read") != TString::npos, logs);
- UNIT_ASSERT_C(logs.find("was successfully fetched") == TString::npos, logs);
- return f;
- }
-
- Y_UNIT_TEST(Service) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(OK_CLIENT);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(GetCachePath());
- TTvmClient f = GetClient(s);
-
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(2524608000), f.GetUpdateTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), f.GetUpdateTimeOfServiceTickets());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(2525126400), f.GetInvalidationTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), f.GetInvalidationTimeOfServiceTickets());
-
- UNIT_ASSERT(f.CheckServiceTicket(SRV_TICKET_123));
- UNIT_ASSERT_EXCEPTION(f.CheckUserTicket(PROD_TICKET), yexception);
- UNIT_ASSERT_EXCEPTION(f.CheckUserTicket(TEST_TICKET), yexception);
- }
-
- Y_UNIT_TEST(User) {
- NTvmApi::TClientSettings s;
- s.EnableUserTicketChecking(EBlackboxEnv::Prod);
- s.SetDiskCacheDir(GetCachePath());
-
- TTvmClient f = GetClient(s);
- UNIT_ASSERT_EXCEPTION(f.CheckServiceTicket(SRV_TICKET_123), yexception);
- UNIT_ASSERT(f.CheckUserTicket(PROD_TICKET));
- UNIT_ASSERT(!f.CheckUserTicket(TEST_TICKET));
- }
-
- Y_UNIT_TEST(Ctors) {
- NTvmApi::TClientSettings s;
- s.EnableUserTicketChecking(EBlackboxEnv::Prod);
- s.SetDiskCacheDir(GetCachePath());
-
- TTvmClient f = GetClient(s);
- f = GetClient(s);
-
- TVector<TTvmClient> v;
- v.push_back(std::move(f));
- v.front() = std::move(*v.begin());
- }
-
- Y_UNIT_TEST(Tickets) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(OK_CLIENT);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(GetCachePath());
- TTvmClient f = GetClient(s);
-
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), f.GetUpdateTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::ParseIso8601("2050-01-01T00:00:00.000000Z"), f.GetUpdateTimeOfServiceTickets());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), f.GetInvalidationTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(std::numeric_limits<size_t>::max()), f.GetInvalidationTimeOfServiceTickets());
-
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIKhCUkQY:CX", f.GetServiceTicketFor("blackbox"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIKhCUkQY:CX", f.GetServiceTicketFor(19));
- UNIT_ASSERT_EXCEPTION_CONTAINS(f.GetServiceTicketFor("blackbox2"),
- TBrokenTvmClientSettings,
- "Destination 'blackbox2' was not specified in settings. Check your settings (if you use Qloud/YP/tvmtool - check it's settings)");
- UNIT_ASSERT_EXCEPTION_CONTAINS(f.GetServiceTicketFor(20),
- TBrokenTvmClientSettings,
- "Destination '20' was not specified in settings. Check your settings (if you use Qloud/YP/tvmtool - check it's settings)");
- }
-
- Y_UNIT_TEST(Tool) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
-
- NTvmTool::TClientSettings s("push-client");
- s.SetPort(port);
- s.SetAuthToken(AUTH_TOKEN);
- auto l = MakeIntrusive<TLogger>();
- {
- TTvmClient f(s, l);
-
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(14380887840), f.GetUpdateTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(14380887840), f.GetUpdateTimeOfServiceTickets());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(14381406240), f.GetInvalidationTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(std::numeric_limits<time_t>::max()), f.GetInvalidationTimeOfServiceTickets());
-
- UNIT_ASSERT_VALUES_EQUAL(SERVICE_TICKET_PC, f.GetServiceTicketFor("pass_likers"));
- UNIT_ASSERT_VALUES_EQUAL(SERVICE_TICKET_PC, f.GetServiceTicketFor(100502));
- UNIT_ASSERT_EXCEPTION_CONTAINS(f.GetServiceTicketFor("blackbox"),
- TBrokenTvmClientSettings,
- "Destination 'blackbox' was not specified in settings. Check your settings (if you use Qloud/YP/tvmtool - check it's settings)");
- UNIT_ASSERT_EXCEPTION_CONTAINS(f.GetServiceTicketFor(242),
- TBrokenTvmClientSettings,
- "Destination '242' was not specified in settings. Check your settings (if you use Qloud/YP/tvmtool - check it's settings)");
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100501, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:100502)]\n"
- << "7: Tickets fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Public keys fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(CheckRoles) {
- { // roles not configured
- TTvmClient f(new TMockedUpdater(TMockedUpdater::TSettings{
- .SelfTvmId = OK_CLIENT,
- }));
-
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok,
- f.CheckServiceTicket(SRV_TICKET_123).GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok,
- f.CheckServiceTicket(SRV_TICKET_456).GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Malformed,
- f.CheckServiceTicket("asdfg").GetStatus());
- }
-
- { // roles configured
- NRoles::TRolesPtr roles = std::make_shared<NRoles::TRoles>(
- NRoles::TRoles::TMeta{},
- NRoles::TRoles::TTvmConsumers{
- {123, std::make_shared<NRoles::TConsumerRoles>(
- THashMap<TString, NRoles::TEntitiesPtr>())},
- },
- NRoles::TRoles::TUserConsumers{},
- std::make_shared<TString>());
- TTvmClient f(new TMockedUpdater(TMockedUpdater::TSettings{
- .SelfTvmId = OK_CLIENT,
- .Roles = roles,
- }));
-
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok,
- f.CheckServiceTicket(SRV_TICKET_123).GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::NoRoles,
- f.CheckServiceTicket(SRV_TICKET_456).GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Malformed,
- f.CheckServiceTicket("asdfg").GetStatus());
- }
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/files/ok.cache b/library/cpp/tvmauth/client/ut/files/ok.cache
deleted file mode 100644
index 768d4953d1..0000000000
--- a/library/cpp/tvmauth/client/ut/files/ok.cache
+++ /dev/null
Binary files differ
diff --git a/library/cpp/tvmauth/client/ut/files/public_keys b/library/cpp/tvmauth/client/ut/files/public_keys
deleted file mode 100644
index fa683d18f3..0000000000
--- a/library/cpp/tvmauth/client/ut/files/public_keys
+++ /dev/null
Binary files differ
diff --git a/library/cpp/tvmauth/client/ut/files/roles b/library/cpp/tvmauth/client/ut/files/roles
deleted file mode 100644
index 36864ae50a..0000000000
--- a/library/cpp/tvmauth/client/ut/files/roles
+++ /dev/null
Binary files differ
diff --git a/library/cpp/tvmauth/client/ut/files/service_tickets b/library/cpp/tvmauth/client/ut/files/service_tickets
deleted file mode 100644
index 7a6985a34d..0000000000
--- a/library/cpp/tvmauth/client/ut/files/service_tickets
+++ /dev/null
Binary files differ
diff --git a/library/cpp/tvmauth/client/ut/last_error_ut.cpp b/library/cpp/tvmauth/client/ut/last_error_ut.cpp
deleted file mode 100644
index 6751e78be7..0000000000
--- a/library/cpp/tvmauth/client/ut/last_error_ut.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-#include <library/cpp/tvmauth/client/misc/last_error.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(LastError) {
- Y_UNIT_TEST(common) {
- TLastError le;
-
- UNIT_ASSERT_VALUES_EQUAL("OK",
- le.GetLastError(true));
- UNIT_ASSERT_VALUES_EQUAL("Internal client error: failed to collect last useful error message, please report this message to tvm-dev@yandex-team.ru",
- le.GetLastError(false));
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(le.ThrowLastError(),
- TNonRetriableException,
- "Internal client error: failed to collect last useful error message");
-
- le.ProcessError(TLastError::EType::Retriable, TLastError::EScope::PublicKeys, "err_re#1");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_re#1",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::Retriable, TLastError::EScope::PublicKeys, "err_re#2");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_re#2",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::NonRetriable, TLastError::EScope::PublicKeys, "err_nonre#3");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#3",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::NonRetriable, TLastError::EScope::PublicKeys, "err_nonre#4");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#4",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::Retriable, TLastError::EScope::PublicKeys, "err_re#5");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#4",
- le.GetLastError(false));
- UNIT_ASSERT_EXCEPTION_CONTAINS(le.ThrowLastError(),
- TNonRetriableException,
- "Failed to start TvmClient. Do not retry: PublicKeys: err_nonre#4");
-
- le.ProcessError(TLastError::EType::Retriable, TLastError::EScope::ServiceTickets, "err_re#6");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#4",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::Retriable, TLastError::EScope::ServiceTickets, "err_re#7");
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#4",
- le.GetLastError(false));
- le.ProcessError(TLastError::EType::NonRetriable, TLastError::EScope::ServiceTickets, "err_nonre#8");
- UNIT_ASSERT_VALUES_EQUAL("ServiceTickets: err_nonre#8",
- le.GetLastError(false));
-
- le.ClearError(TLastError::EScope::ServiceTickets);
- UNIT_ASSERT_VALUES_EQUAL("PublicKeys: err_nonre#4",
- le.GetLastError(false));
- le.ClearError(TLastError::EScope::PublicKeys);
- UNIT_ASSERT_VALUES_EQUAL("Internal client error: failed to collect last useful error message, please report this message to tvm-dev@yandex-team.ru",
- le.GetLastError(false));
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/logger_ut.cpp b/library/cpp/tvmauth/client/ut/logger_ut.cpp
deleted file mode 100644
index 76236e8913..0000000000
--- a/library/cpp/tvmauth/client/ut/logger_ut.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/logger.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientLogger) {
- int i = 0;
-
- Y_UNIT_TEST(Debug) {
- TLogger l;
- l.Debug("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("7: qwerty\n", l.Stream.Str());
- }
-
- Y_UNIT_TEST(Info) {
- TLogger l;
- l.Info("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("6: qwerty\n", l.Stream.Str());
- }
-
- Y_UNIT_TEST(Warning) {
- TLogger l;
- l.Warning("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("4: qwerty\n", l.Stream.Str());
- }
-
- Y_UNIT_TEST(Error) {
- TLogger l;
- l.Error("qwerty");
- UNIT_ASSERT_VALUES_EQUAL("3: qwerty\n", l.Stream.Str());
- }
-
-#ifdef _unix_
- Y_UNIT_TEST(Cerr_) {
- TCerrLogger l(5);
- l.Error("hit");
- l.Debug("miss");
- }
-#endif
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/decoder_ut.cpp b/library/cpp/tvmauth/client/ut/roles/decoder_ut.cpp
deleted file mode 100644
index 0ee5fc7cb7..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/decoder_ut.cpp
+++ /dev/null
@@ -1,163 +0,0 @@
-#include <library/cpp/tvmauth/client/exception.h>
-#include <library/cpp/tvmauth/client/misc/roles/decoder.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NRoles;
-
-Y_UNIT_TEST_SUITE(Decoder) {
- const TString BROTLI = NUtils::Base64url2bin("GyMAAAR0Y6ku58ObclAQzDweUSUwbdqc5yOOKgI");
- const TString GZIP = NUtils::Base64url2bin("H4sIAAAAAAAA_yrOz01VKEstqkTGCpm5BflFJYl5JQpJOflJgAAAAP__MbeeiSQAAAA");
- const TString ZSTD = NUtils::Base64url2bin("KLUv_QBY9AAAwHNvbWUgdmVyeSBpbXBvcnRhbnQgYmxvYgEAc-4IAQAA");
-
- Y_UNIT_TEST(Decode) {
- // Errs
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::Decode(
- "1:brotli:10000:88839244E8C7C426B20729AF1A13AD792C5FA83C7F2FB6ADCFC60DA1B5EF9603",
- TString(BROTLI)),
- yexception,
- "Decoded blob has bad size: expected 10000, actual 36");
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::Decode(
- "1:brotli:36:88839244E8C7C426B20729AF1A13AD792C5FA83C7F2FB6ADCFC60DA1B5EF0000",
- TString(BROTLI)),
- yexception,
- "Decoded blob has bad sha256");
-
- // OK
- TString decoded;
- UNIT_ASSERT_NO_EXCEPTION(
- decoded = TDecoder::Decode("", "some veryveryveryvery important blob"));
- UNIT_ASSERT_VALUES_EQUAL(decoded, "some veryveryveryvery important blob");
-
- UNIT_ASSERT_NO_EXCEPTION(
- decoded = TDecoder::Decode(
- "1:brotli:36:88839244E8C7C426B20729AF1A13AD792C5FA83C7F2FB6ADCFC60DA1B5EF9603",
- TString(BROTLI)));
- UNIT_ASSERT_VALUES_EQUAL(decoded, "some veryveryveryvery important blob");
-
- UNIT_ASSERT_NO_EXCEPTION(
- decoded = TDecoder::Decode(
- "1:gzip:36:88839244E8C7C426B20729AF1A13AD792C5FA83C7F2FB6ADCFC60DA1B5EF9603",
- TString(GZIP)));
- UNIT_ASSERT_VALUES_EQUAL(decoded, "some veryveryveryvery important blob");
-
- UNIT_ASSERT_NO_EXCEPTION(
- decoded = TDecoder::Decode(
- "1:zstd:36:88839244E8C7C426B20729AF1A13AD792C5FA83C7F2FB6ADCFC60DA1B5EF9603",
- TString(ZSTD)));
- UNIT_ASSERT_VALUES_EQUAL(decoded, "some veryveryveryvery important blob");
- }
-
- Y_UNIT_TEST(UnknownCodecs) {
- for (const TStringBuf codec : {"lz", "lzma", "kek"}) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::DecodeImpl(codec, ""),
- yexception,
- TStringBuilder() << "unknown codec: '" << codec << "'");
- }
- }
-
- Y_UNIT_TEST(ParseCodec) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::ParseCodec("2:kek"),
- yexception,
- "unknown codec format version; known: 1; got: 2");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::ParseCodec("1:::"),
- yexception,
- "codec type is empty");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::ParseCodec("1:some_codec:asd:"),
- yexception,
- "decoded blob size is not number");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::ParseCodec("1:some_codec:789:qwe"),
- yexception,
- "sha256 of decoded blob has invalid length: expected 64, got 3");
-
- TDecoder::TCodecInfo info;
- UNIT_ASSERT_NO_EXCEPTION(
- info = TDecoder::ParseCodec("1:some_codec:789:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
-
- UNIT_ASSERT_VALUES_EQUAL("some_codec", info.Type);
- UNIT_ASSERT_VALUES_EQUAL(789, info.Size);
- UNIT_ASSERT_VALUES_EQUAL("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
- info.Sha256);
- }
-
- Y_UNIT_TEST(DecodeBrolti) {
- UNIT_ASSERT_EXCEPTION(
- TDecoder::DecodeBrolti(""),
- yexception);
-
- TString blob;
- UNIT_ASSERT_NO_EXCEPTION(
- blob = TDecoder::DecodeBrolti(
- TString(BROTLI)));
-
- UNIT_ASSERT_VALUES_EQUAL(
- "some veryveryveryvery important blob",
- blob);
- }
-
- Y_UNIT_TEST(DecodeGzip) {
- TString blob;
- UNIT_ASSERT_NO_EXCEPTION(blob = TDecoder::DecodeGzip(""));
- UNIT_ASSERT_VALUES_EQUAL("", blob);
-
- UNIT_ASSERT_NO_EXCEPTION(
- blob = TDecoder::DecodeGzip(
- TString(GZIP)));
-
- UNIT_ASSERT_VALUES_EQUAL(
- "some veryveryveryvery important blob",
- blob);
- }
-
- Y_UNIT_TEST(DecodeZstd) {
- TString blob;
- UNIT_ASSERT_NO_EXCEPTION(blob = TDecoder::DecodeZstd(""));
- UNIT_ASSERT_VALUES_EQUAL("", blob);
-
- UNIT_ASSERT_NO_EXCEPTION(
- blob = TDecoder::DecodeZstd(
- TString(ZSTD)));
-
- UNIT_ASSERT_VALUES_EQUAL(
- "some veryveryveryvery important blob",
- blob);
- }
-
- Y_UNIT_TEST(VerifySize) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::VerifySize("qwerty", 100),
- yexception,
- TStringBuilder() << "Decoded blob has bad size: expected 100, actual 6");
-
- UNIT_ASSERT_NO_EXCEPTION(TDecoder::VerifySize("qwert", 5));
- }
-
- Y_UNIT_TEST(VerifyChecksum) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TDecoder::VerifyChecksum("qwerty", "zzzz"),
- yexception,
- "Decoded blob has bad sha256: expected=zzzz,"
- " actual=65E84BE33532FB784C48129675F9EFF3A682B27168C0EA744B2CF58EE02337C5");
-
- UNIT_ASSERT_NO_EXCEPTION(
- TDecoder::VerifyChecksum("qwerty",
- "65E84BE33532FB784C48129675F9EFF3A682B27168C0EA744B2CF58EE02337C5"));
- UNIT_ASSERT_NO_EXCEPTION(
- TDecoder::VerifyChecksum("qwerty",
- "65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5"));
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/entities_index_ut.cpp b/library/cpp/tvmauth/client/ut/roles/entities_index_ut.cpp
deleted file mode 100644
index 7e62a87b64..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/entities_index_ut.cpp
+++ /dev/null
@@ -1,358 +0,0 @@
-#include <library/cpp/tvmauth/client/ut/common.h>
-
-#include <library/cpp/tvmauth/client/misc/roles/entities_index.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <array>
-
-using namespace NTvmAuth::NRoles;
-
-Y_UNIT_TEST_SUITE(RolesEntitiesIndex) {
- Y_UNIT_TEST(Stage) {
- TEntitiesIndex::TStage stage({
- "key#1",
- "key#2",
- "key#3",
- "key#4",
- });
-
- const std::vector<std::vector<TString>> results = {
- {"key#1"},
- {"key#2"},
- {"key#1", "key#2"},
- {"key#3"},
- {"key#1", "key#3"},
- {"key#2", "key#3"},
- {"key#1", "key#2", "key#3"},
- {"key#4"},
- {"key#1", "key#4"},
- {"key#2", "key#4"},
- {"key#1", "key#2", "key#4"},
- {"key#3", "key#4"},
- {"key#1", "key#3", "key#4"},
- {"key#2", "key#3", "key#4"},
- {"key#1", "key#2", "key#3", "key#4"},
- };
-
- std::vector<TString> keys;
- for (const std::vector<TString>& res : results) {
- UNIT_ASSERT(stage.GetNextKeySet(keys));
- UNIT_ASSERT_VALUES_EQUAL(keys, res);
- }
-
- UNIT_ASSERT_C(!stage.GetNextKeySet(keys), keys);
- }
-
- Y_UNIT_TEST(GetUniqueSortedKeys) {
- std::vector<TEntityPtr> entities;
-
- UNIT_ASSERT_VALUES_EQUAL(std::set<TString>(),
- TEntitiesIndex::GetUniqueSortedKeys(entities));
-
- entities = {
- std::make_shared<TEntity>(),
- };
- UNIT_ASSERT_VALUES_EQUAL(std::set<TString>(),
- TEntitiesIndex::GetUniqueSortedKeys(entities));
-
- entities = {
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#1"},
- }),
- };
- UNIT_ASSERT_VALUES_EQUAL(std::set<TString>({
- "key#1",
- }),
- TEntitiesIndex::GetUniqueSortedKeys(entities));
-
- entities = {
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#1"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- }),
- };
- UNIT_ASSERT_VALUES_EQUAL(std::set<TString>({
- "key#1",
- "key#2",
- }),
- TEntitiesIndex::GetUniqueSortedKeys(entities));
- }
-
- Y_UNIT_TEST(MakeUnique) {
- const TEntityPtr entityA = std::make_shared<TEntity>(TEntity{{"key#1", "aaaa"}});
- const TEntityPtr entityA2 = std::make_shared<TEntity>(TEntity{{"key#1", "aaaa"}});
- const TEntityPtr entityB = std::make_shared<TEntity>(TEntity{{"key#1", "bbbb"}});
-
- TEntitiesIndex::TSubTree idx = {
- std::vector<TEntityPtr>{
- entityA,
- entityA,
- },
- TEntitiesIndex::TIdxByAttrs{
- {
- TKeyValue{"key#1", "value#11"},
- TEntitiesIndex::TSubTree{
- std::vector<TEntityPtr>{
- entityA,
- entityB,
- entityA,
- },
- TEntitiesIndex::TIdxByAttrs{
- {
- TKeyValue{"key#2", "value#21"},
- TEntitiesIndex::TSubTree{
- std::vector<TEntityPtr>{
- entityA,
- entityB,
- entityA,
- },
- TEntitiesIndex::TIdxByAttrs{},
- },
- },
- },
- },
- },
- {
- TKeyValue{"key#1", "value#12"},
- TEntitiesIndex::TSubTree{
- std::vector<TEntityPtr>{
- entityA,
- entityB,
- entityA2,
- },
- TEntitiesIndex::TIdxByAttrs{},
- },
- },
- },
- };
-
- TEntitiesIndex::MakeUnique(idx);
-
- UNIT_ASSERT_VALUES_EQUAL(idx.Entities.size(), 1);
-
- auto it = idx.SubTree.find(TKeyValue{"key#1", "value#12"});
- UNIT_ASSERT(it != idx.SubTree.end());
- UNIT_ASSERT_VALUES_EQUAL(it->second.Entities.size(), 2);
-
- it = idx.SubTree.find(TKeyValue{"key#1", "value#11"});
- UNIT_ASSERT(it != idx.SubTree.end());
- UNIT_ASSERT_VALUES_EQUAL(it->second.Entities.size(), 2);
-
- it = it->second.SubTree.find(TKeyValue{"key#2", "value#21"});
- UNIT_ASSERT(it != it->second.SubTree.end());
- UNIT_ASSERT_VALUES_EQUAL(it->second.Entities.size(), 2);
- }
-
- Y_UNIT_TEST(GetByAttrs) {
- const TEntitiesIndex index = CreateEntitiesIndex();
-
- UNIT_ASSERT_STRINGS_EQUAL(
- index.PrintDebugString(),
- R"(
-"key#1/value#11"
- "key#2/value#22"
- "key#3/value#33"
- "key#2/value#23"
- "key#3/value#33"
- "key#3/value#33"
-"key#1/value#13"
- "key#3/value#33"
-"key#2/value#22"
- "key#3/value#33"
-"key#2/value#23"
- "key#3/value#33"
-"key#3/value#33"
-)");
-
- struct TCase {
- TEntity AttrsToFind;
- std::vector<TEntity> Result;
- };
-
- std::vector<TCase> cases = {
- {
- TEntity{},
- std::vector<TEntity>{
- TEntity{
- {"key#1", "value#11"},
- },
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- },
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- },
- TEntity{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- },
- },
- },
- {
- TEntity{
- {"key#1", "value#11"},
- },
- std::vector<TEntity>{
- TEntity{
- {"key#1", "value#11"},
- },
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- },
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- },
- },
- },
- {
- TEntity{
- {"key#1", "value#13"},
- },
- std::vector<TEntity>{
- TEntity{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- },
- },
- },
- {
- TEntity{
- {"key#1", "value#14"},
- },
- std::vector<TEntity>{},
- },
- {
- TEntity{
- {"key#2", "value#22"},
- },
- std::vector<TEntity>{
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- },
- },
- },
- {
- TEntity{
- {"key#3", "value#33"},
- },
- std::vector<TEntity>{
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- },
- TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- },
- TEntity{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- },
- },
- },
- };
-
- for (const TCase& c : cases) {
- std::vector<TEntityPtr> expected;
- for (const TEntity& e : c.Result) {
- expected.push_back(std::make_shared<TEntity>(e));
- }
-
- UNIT_ASSERT_VALUES_EQUAL_C(
- index.GetEntitiesWithAttrs(c.AttrsToFind.begin(), c.AttrsToFind.end()),
- expected,
- "'" << c.AttrsToFind << "'");
- }
- }
-
- Y_UNIT_TEST(Contains) {
- const TEntitiesIndex index = CreateEntitiesIndex();
-
- struct TCase {
- TEntity Exact;
- bool Result = false;
- };
-
- std::vector<TCase> cases = {
- {
- TEntity{},
- false,
- },
- {
- TEntity{
- {"key#1", "value#11"},
- },
- true,
- },
- {
- TEntity{
- {"key#1", "value#13"},
- },
- false,
- },
- {
- TEntity{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- },
- true,
- },
- };
-
- for (const TCase& c : cases) {
- UNIT_ASSERT_VALUES_EQUAL_C(
- index.ContainsExactEntity(c.Exact.begin(), c.Exact.end()),
- c.Result,
- "'" << c.Exact << "'");
- }
- }
-}
-
-template <>
-void Out<std::vector<TString>>(IOutputStream& o, const std::vector<TString>& s) {
- for (const auto& key : s) {
- o << key << ",";
- }
-}
-
-template <>
-void Out<std::set<TString>>(IOutputStream& o, const std::set<TString>& s) {
- for (const auto& key : s) {
- o << key << ",";
- }
-}
-
-template <>
-void Out<std::vector<TEntityPtr>>(IOutputStream& o, const std::vector<TEntityPtr>& v) {
- for (const TEntityPtr& p : v) {
- o << *p << Endl;
- }
-}
-
-template <>
-void Out<TEntityPtr>(IOutputStream& o, const TEntityPtr& v) {
- o << *v;
-}
-
-template <>
-void Out<TEntity>(IOutputStream& o, const TEntity& v) {
- for (const auto& [key, value] : v) {
- o << key << "->" << value << Endl;
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/parser_ut.cpp b/library/cpp/tvmauth/client/ut/roles/parser_ut.cpp
deleted file mode 100644
index 4a2afac483..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/parser_ut.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-#include <library/cpp/tvmauth/client/misc/roles/parser.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/json/json_reader.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NRoles;
-
-Y_UNIT_TEST_SUITE(Parser) {
- static NJson::TJsonValue ToJsonValue(TStringBuf body) {
- NJson::TJsonValue doc;
- UNIT_ASSERT(NJson::ReadJsonTree(body, &doc));
- return doc;
- }
-
- Y_UNIT_TEST(GetEntity) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetEntity(ToJsonValue(R"({"scope": false})"),
- "cons",
- "read"),
- yexception,
- "entity is map (str->str), got value Boolean. consumer 'cons' with role 'read'");
-
- TEntityPtr en;
- UNIT_ASSERT_NO_EXCEPTION(
- en = TParser::GetEntity(ToJsonValue(R"({})"),
- "cons",
- "read"));
- UNIT_ASSERT_VALUES_EQUAL(en->size(), 0);
-
- UNIT_ASSERT_NO_EXCEPTION(
- en = TParser::GetEntity(ToJsonValue(R"({"key1": "val1", "key2": "val2"})"),
- "cons",
- "read"));
- UNIT_ASSERT_VALUES_EQUAL(en->size(), 2);
- }
-
- Y_UNIT_TEST(GetEntities) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetEntities(ToJsonValue(R"([{},[]])"),
- "cons",
- "read"),
- yexception,
- "role entity for role must be map: consumer 'cons' with role 'read' has Array");
-
- TEntitiesPtr en;
- UNIT_ASSERT_NO_EXCEPTION(
- en = TParser::GetEntities(ToJsonValue(R"([])"),
- "cons",
- "read"));
- UNIT_ASSERT(!en->Contains({}));
-
- UNIT_ASSERT_NO_EXCEPTION(
- en = TParser::GetEntities(ToJsonValue(R"([{}])"),
- "cons",
- "read"));
- UNIT_ASSERT(en->Contains({}));
- }
-
- Y_UNIT_TEST(GetConsumer) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumer(ToJsonValue(R"({"role1": [],"role2": {}})"),
- "cons"),
- yexception,
- "entities for roles must be array: 'role2' is Map");
-
- TConsumerRolesPtr c;
- UNIT_ASSERT_NO_EXCEPTION(
- c = TParser::GetConsumer(ToJsonValue(R"({"role1": [],"role2": []})"),
- "cons"));
- UNIT_ASSERT_EQUAL(c->GetRoles().size(), 2);
- UNIT_ASSERT(c->HasRole("role1"));
- UNIT_ASSERT(c->HasRole("role2"));
- UNIT_ASSERT(!c->HasRole("role3"));
- }
-
- Y_UNIT_TEST(GetConsumers) {
- TRoles::TTvmConsumers cons;
- UNIT_ASSERT_NO_EXCEPTION(
- cons = TParser::GetConsumers<TTvmId>(ToJsonValue(R"({})"),
- "tvm"));
- UNIT_ASSERT_VALUES_EQUAL(0, cons.size());
-
- UNIT_ASSERT_NO_EXCEPTION(
- cons = TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {}})"),
- "tvm"));
- UNIT_ASSERT_VALUES_EQUAL(0, cons.size());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": []})"),
- "tvm"),
- yexception,
- "'tvm' must be object");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {"asd": []}})"),
- "tvm"),
- yexception,
- "roles for consumer must be map: 'asd' is Array");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {"asd": {}}})"),
- "tvm"),
- yexception,
- "id must be valid positive number of proper size for tvm. got 'asd'");
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {"1120000000001062": {}}})"),
- "tvm"),
- yexception,
- "id must be valid positive number of proper size for tvm. got '1120000000001062'");
- UNIT_ASSERT_NO_EXCEPTION(
- TParser::GetConsumers<TUid>(ToJsonValue(R"({"user": {"1120000000001062": {}}})"),
- "user"));
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {"42": {}, "042": {}}})"),
- "tvm"),
- yexception,
- "consumer duplicate detected: '42' for tvm");
-
- UNIT_ASSERT_NO_EXCEPTION(
- cons = TParser::GetConsumers<TTvmId>(ToJsonValue(R"({"tvm": {"42": {}}})"),
- "tvm"));
- UNIT_ASSERT_VALUES_EQUAL(1, cons.size());
- }
-
- Y_UNIT_TEST(GetMeta) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetMeta(ToJsonValue(R"({})")),
- yexception,
- "Missing 'revision'");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetMeta(ToJsonValue(R"({"revision": null})")),
- yexception,
- "'revision' has unexpected type: Null");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetMeta(ToJsonValue(R"({"revision": 100500})")),
- yexception,
- "Missing 'born_date'");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TParser::GetMeta(ToJsonValue(R"({"revision": 100500, "born_date": false})")),
- yexception,
- "key 'born_date' must be uint");
-
- TRoles::TMeta meta;
- UNIT_ASSERT_NO_EXCEPTION(
- meta = TParser::GetMeta(ToJsonValue(R"({"revision": 100500, "born_date": 42})")));
- UNIT_ASSERT_VALUES_EQUAL("100500", meta.Revision);
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(42), meta.BornTime);
-
- UNIT_ASSERT_NO_EXCEPTION(
- meta = TParser::GetMeta(ToJsonValue(R"({"revision": "100501", "born_date": 42})")));
- UNIT_ASSERT_VALUES_EQUAL("100501", meta.Revision);
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(42), meta.BornTime);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/roles_ut.cpp b/library/cpp/tvmauth/client/ut/roles/roles_ut.cpp
deleted file mode 100644
index d485dd857a..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/roles_ut.cpp
+++ /dev/null
@@ -1,419 +0,0 @@
-#include <library/cpp/tvmauth/client/ut/common.h>
-
-#include <library/cpp/tvmauth/client/exception.h>
-#include <library/cpp/tvmauth/client/misc/roles/roles.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <array>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NRoles;
-
-Y_UNIT_TEST_SUITE(Roles) {
- Y_UNIT_TEST(EntContains) {
- TEntities ent(CreateEntitiesIndex());
-
- UNIT_ASSERT(ent.Contains({{"key#1", "value#11"}}));
- UNIT_ASSERT(ent.Contains({
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- }));
- UNIT_ASSERT(!ent.Contains({{"key#111", "value#11"}}));
- UNIT_ASSERT(!ent.Contains({
- {"key#111", "value#13"},
- {"key#3", "value#33"},
- }));
-
- // valid calls
- {
- std::array<const std::pair<TStringBuf, TString>, 1> arr = {{{"key#1", "value#11"}}};
- UNIT_ASSERT(ent.ContainsSortedUnique<TStringBuf>({arr.begin(), arr.end()}));
- }
- {
- std::array<const std::pair<TString, TStringBuf>, 2> arr = {{
- {"key#1", "value#13"},
- {"key#3", "value#33"},
- }};
- bool res = ent.ContainsSortedUnique<TString, TStringBuf>({arr.begin(), arr.end()});
- UNIT_ASSERT(res);
- }
- {
- std::array<const std::pair<TStringBuf, TStringBuf>, 1> arr = {{{"key#111", "value#11"}}};
- bool res = ent.ContainsSortedUnique<TStringBuf, TStringBuf>({arr.begin(), arr.end()});
- UNIT_ASSERT(!res);
- }
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#111", "value#13"},
- {"key#3", "value#33"},
- }};
- UNIT_ASSERT(!ent.ContainsSortedUnique({arr.begin(), arr.end()}));
- }
-
- // invalid calls
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#3", "value#33"},
- {"key#1", "value#13"},
- }};
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- ent.ContainsSortedUnique({arr.begin(), arr.end()}),
- TIllegalUsage,
- "attrs are not sorted: 'key#3' before 'key#1'");
- }
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#1", "value#13"},
- {"key#1", "value#13"},
- }};
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- ent.ContainsSortedUnique({arr.begin(), arr.end()}),
- TIllegalUsage,
- "attrs are not unique: 'key#1'");
- }
- }
-
- Y_UNIT_TEST(EntWithAttrs) {
- TEntities ent(CreateEntitiesIndex());
-
- UNIT_ASSERT_VALUES_EQUAL(
- ent.GetEntitiesWithAttrs({{"key#1", "value#11"}}),
- std::vector<TEntityPtr>({
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- }),
- }));
- UNIT_ASSERT_VALUES_EQUAL(
- ent.GetEntitiesWithAttrs({{"key#111", "value#11"}}),
- std::vector<TEntityPtr>());
-
- // valid calls
- {
- std::array<const std::pair<TStringBuf, TString>, 2> arr = {{
- {"key#1", "value#11"},
- {"key#3", "value#33"},
- }};
- auto vec = ent.GetEntitiesWithSortedUniqueAttrs<TStringBuf>({arr.begin(), arr.end()});
- UNIT_ASSERT_VALUES_EQUAL(
- vec,
- std::vector<TEntityPtr>({
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#22"},
- {"key#3", "value#33"},
- }),
- std::make_shared<TEntity>(TEntity{
- {"key#1", "value#11"},
- {"key#2", "value#23"},
- {"key#3", "value#33"},
- }),
- }));
- }
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#111", "value#13"},
- {"key#3", "value#33"},
- }};
- UNIT_ASSERT_VALUES_EQUAL(
- ent.GetEntitiesWithSortedUniqueAttrs({arr.begin(), arr.end()}),
- std::vector<TEntityPtr>());
- }
-
- // invalid calls
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#3", "value#33"},
- {"key#1", "value#13"},
- }};
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- ent.GetEntitiesWithSortedUniqueAttrs({arr.begin(), arr.end()}),
- TIllegalUsage,
- "attrs are not sorted: 'key#3' before 'key#1'");
- }
- {
- std::array<const std::pair<TString, TString>, 2> arr = {{
- {"key#1", "value#13"},
- {"key#1", "value#13"},
- }};
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- ent.GetEntitiesWithSortedUniqueAttrs({arr.begin(), arr.end()}),
- TIllegalUsage,
- "attrs are not unique: 'key#1'");
- }
- }
-
- Y_UNIT_TEST(Consumer) {
- TConsumerRoles c({
- {"read", std::make_shared<TEntities>(CreateEntitiesIndex())},
- {"write", std::make_shared<TEntities>(CreateEntitiesIndex())},
- });
-
- UNIT_ASSERT_EQUAL(c.GetRoles().size(), 2);
- UNIT_ASSERT(c.HasRole("read"));
- UNIT_ASSERT(c.HasRole("write"));
- UNIT_ASSERT(!c.HasRole("access"));
-
- UNIT_ASSERT_EQUAL(nullptr, c.GetEntitiesForRole("access"));
-
- TEntitiesPtr ent = c.GetEntitiesForRole("read");
- UNIT_ASSERT_UNEQUAL(nullptr, ent);
- UNIT_ASSERT(ent->Contains({{"key#1", "value#11"}}));
- UNIT_ASSERT(!ent->Contains({{"key#111", "value#11"}}));
-
- UNIT_ASSERT(c.CheckRoleForExactEntity("read", {{"key#1", "value#11"}}));
- UNIT_ASSERT(!c.CheckRoleForExactEntity("access", {{"key#1", "value#11"}}));
- UNIT_ASSERT(!c.CheckRoleForExactEntity("read", {{"key#111", "value#11"}}));
- UNIT_ASSERT(!c.CheckRoleForExactEntity("read", {}));
- }
-
- Y_UNIT_TEST(RolesService) {
- TRoles r(
- {},
- {
- {100500, std::make_shared<TConsumerRoles>(TEntitiesByRoles{
- {"write", std::make_shared<TEntities>(CreateEntitiesIndex())},
- })},
- },
- {},
- std::make_shared<TString>());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- r.GetRolesForService(NUnittest::CreateServiceTicket(
- ETicketStatus::InvalidDst,
- 100500)),
- TIllegalUsage,
- "Service ticket must be valid, got: InvalidDst");
-
- TConsumerRolesPtr cons;
- UNIT_ASSERT_NO_EXCEPTION(
- cons = r.GetRolesForService(NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100501)));
- UNIT_ASSERT_EQUAL(nullptr, cons);
-
- cons = r.GetRolesForService(NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500));
- UNIT_ASSERT_UNEQUAL(nullptr, cons);
- UNIT_ASSERT_EQUAL(cons->GetRoles().size(), 1);
- UNIT_ASSERT(!cons->HasRole("read"));
- UNIT_ASSERT(cons->HasRole("write"));
-
- ////shortcuts
- // no tvmid
- UNIT_ASSERT(!r.CheckServiceRole(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100501),
- "write"));
-
- // no role
- UNIT_ASSERT(!r.CheckServiceRole(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500),
- "read"));
-
- // success
- UNIT_ASSERT(r.CheckServiceRole(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500),
- "write"));
-
- // no tvmid
- UNIT_ASSERT(!r.CheckServiceRoleForExactEntity(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100501),
- "write",
- {{"key#1", "value#11"}}));
-
- // no role
- UNIT_ASSERT(!r.CheckServiceRoleForExactEntity(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500),
- "read",
- {{"key#1", "value#11"}}));
-
- // no entity
- UNIT_ASSERT(!r.CheckServiceRoleForExactEntity(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500),
- "write",
- {{"key#111", "value#11"}}));
-
- // success
- UNIT_ASSERT(r.CheckServiceRoleForExactEntity(
- NUnittest::CreateServiceTicket(
- ETicketStatus::Ok,
- 100500),
- "write",
- {{"key#1", "value#11"}}));
- }
-
- Y_UNIT_TEST(RolesUser) {
- TRoles r(
- {},
- {},
- {
- {789654, std::make_shared<TConsumerRoles>(TEntitiesByRoles{
- {"read", std::make_shared<TEntities>(CreateEntitiesIndex())},
- })},
- },
- std::make_shared<TString>("some roles"));
-
- UNIT_ASSERT_VALUES_EQUAL("some roles", r.GetRaw());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Malformed,
- 789654,
- {})),
- TIllegalUsage,
- "User ticket must be valid, got: Malformed");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {}),
- 789123),
- TIllegalUsage,
- "User ticket must be from ProdYateam, got from Test");
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- 789123),
- TIllegalUsage,
- "selectedUid must be in user ticket but it's not: 789123");
-
- TConsumerRolesPtr cons;
- UNIT_ASSERT_NO_EXCEPTION(
- cons = r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789123,
- {},
- {},
- EBlackboxEnv::ProdYateam)));
- UNIT_ASSERT_EQUAL(nullptr, cons);
-
- cons = r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam));
- UNIT_ASSERT_UNEQUAL(nullptr, cons);
- UNIT_ASSERT_EQUAL(cons->GetRoles().size(), 1);
- UNIT_ASSERT(cons->HasRole("read"));
- UNIT_ASSERT(!cons->HasRole("write"));
-
- cons = r.GetRolesForUser(NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789123,
- {},
- {789654, 789741},
- EBlackboxEnv::ProdYateam),
- 789654);
- UNIT_ASSERT_UNEQUAL(nullptr, cons);
- UNIT_ASSERT_EQUAL(cons->GetRoles().size(), 1);
- UNIT_ASSERT(cons->HasRole("read"));
- UNIT_ASSERT(!cons->HasRole("write"));
-
- ////shortcuts
- // no uid
- UNIT_ASSERT(!r.CheckUserRole(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789123,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "read"));
-
- // no role
- UNIT_ASSERT(!r.CheckUserRole(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "wrire"));
-
- // success
- UNIT_ASSERT(r.CheckUserRole(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "read"));
-
- // no uid
- UNIT_ASSERT(!r.CheckUserRoleForExactEntity(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789123,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "read",
- {{"key#1", "value#11"}}));
-
- // no role
- UNIT_ASSERT(!r.CheckUserRoleForExactEntity(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "wrire",
- {{"key#1", "value#11"}}));
-
- // no entity
- UNIT_ASSERT(!r.CheckUserRoleForExactEntity(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "read",
- {{"key#111", "value#11"}}));
-
- // success
- UNIT_ASSERT(r.CheckUserRoleForExactEntity(
- NUnittest::CreateUserTicket(
- ETicketStatus::Ok,
- 789654,
- {},
- {},
- EBlackboxEnv::ProdYateam),
- "read",
- {{"key#1", "value#11"}}));
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/tvmapi_roles_fetcher_ut.cpp b/library/cpp/tvmauth/client/ut/roles/tvmapi_roles_fetcher_ut.cpp
deleted file mode 100644
index 7eaf611e82..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/tvmapi_roles_fetcher_ut.cpp
+++ /dev/null
@@ -1,197 +0,0 @@
-#include <library/cpp/tvmauth/client/ut/common.h>
-
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-#include <library/cpp/tvmauth/client/misc/api/roles_fetcher.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/stream/file.h>
-#include <util/system/fs.h>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NTvmApi;
-
-Y_UNIT_TEST_SUITE(TvmApiRolesFetcher) {
- static const TString ROLES = R"({"revision": "100501", "born_date": 42})";
-
- static const TString CACHE_DIR = "./tmp/";
-
- static void CleanCache() {
- NFs::RemoveRecursive(CACHE_DIR);
- NFs::MakeDirectoryRecursive(CACHE_DIR);
- }
-
- Y_UNIT_TEST(ReadFromDisk) {
- CleanCache();
- auto logger = MakeIntrusive<TLogger>();
-
- TRolesFetcherSettings s;
- s.CacheDir = CACHE_DIR;
- s.SelfTvmId = 111111;
- s.IdmSystemSlug = "fem\tida";
- TRolesFetcher fetcher(s, logger);
-
- UNIT_ASSERT(!fetcher.AreRolesOk());
-
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), fetcher.ReadFromDisk());
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: File './tmp/roles' does not exist\n",
- logger->Stream.Str());
- logger->Stream.clear();
-
- const TInstant now = TInstant::Seconds(TInstant::Now().Seconds());
-
- TDiskWriter wr(CACHE_DIR + "roles");
- UNIT_ASSERT(wr.Write("kek", now));
- UNIT_ASSERT_NO_EXCEPTION(fetcher.ReadFromDisk());
- UNIT_ASSERT(!fetcher.AreRolesOk());
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/roles' was successfully read\n"
- << "4: Roles in disk cache are for another slug (kek). Self=fem\tida\n",
- logger->Stream.Str());
- logger->Stream.clear();
-
- UNIT_ASSERT(wr.Write(TRolesFetcher::PrepareDiskFormat(ROLES, "femida_test"), now));
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), fetcher.ReadFromDisk());
- UNIT_ASSERT(!fetcher.AreRolesOk());
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/roles' was successfully read\n"
- "4: Roles in disk cache are for another slug (femida_test). Self=fem\tida\n",
- logger->Stream.Str());
- logger->Stream.clear();
-
- UNIT_ASSERT(wr.Write(TRolesFetcher::PrepareDiskFormat(ROLES, "fem\tida"), now));
- UNIT_ASSERT_VALUES_EQUAL(now, fetcher.ReadFromDisk());
- UNIT_ASSERT(fetcher.AreRolesOk());
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/roles' was successfully read\n"
- "7: Succeed to read roles with revision 100501 from ./tmp/roles\n",
- logger->Stream.Str());
- logger->Stream.clear();
- }
-
- Y_UNIT_TEST(IsTimeToUpdate) {
- TRetrySettings settings;
- settings.RolesUpdatePeriod = TDuration::Minutes(123);
-
- UNIT_ASSERT(!TRolesFetcher::IsTimeToUpdate(settings, TDuration::Seconds(5)));
- UNIT_ASSERT(TRolesFetcher::IsTimeToUpdate(settings, TDuration::Hours(5)));
- }
-
- Y_UNIT_TEST(ShouldWarn) {
- TRetrySettings settings;
- settings.RolesWarnPeriod = TDuration::Minutes(123);
-
- UNIT_ASSERT(!TRolesFetcher::ShouldWarn(settings, TDuration::Seconds(5)));
- UNIT_ASSERT(TRolesFetcher::ShouldWarn(settings, TDuration::Hours(5)));
- }
-
- Y_UNIT_TEST(Update) {
- CleanCache();
- auto logger = MakeIntrusive<TLogger>();
-
- TRolesFetcherSettings s;
- s.CacheDir = CACHE_DIR;
- s.SelfTvmId = 111111;
- TRolesFetcher fetcher(s, logger);
-
- UNIT_ASSERT(!fetcher.AreRolesOk());
-
- NUtils::TFetchResult fetchResult;
- fetchResult.Code = 304;
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- fetcher.Update(NUtils::TFetchResult(fetchResult)),
- yexception,
- "tirole did not return any roles because current roles are actual, but there are no roles in memory");
- UNIT_ASSERT(!fetcher.AreRolesOk());
- UNIT_ASSERT(!NFs::Exists(CACHE_DIR + "roles"));
-
- fetchResult.Code = 206;
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- fetcher.Update(NUtils::TFetchResult(fetchResult)),
- yexception,
- "Unexpected code from tirole: 206.");
- UNIT_ASSERT(!fetcher.AreRolesOk());
- UNIT_ASSERT(!NFs::Exists(CACHE_DIR + "roles"));
-
- fetchResult.Code = 200;
- fetchResult.Response = "kek";
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- fetcher.Update(NUtils::TFetchResult(fetchResult)),
- yexception,
- "Invalid json. 'kek'");
- UNIT_ASSERT(!fetcher.AreRolesOk());
- UNIT_ASSERT(!NFs::Exists(CACHE_DIR + "roles"));
-
- fetchResult.Response = ROLES;
- UNIT_ASSERT_NO_EXCEPTION(fetcher.Update(NUtils::TFetchResult(fetchResult)));
- UNIT_ASSERT(fetcher.AreRolesOk());
- UNIT_ASSERT(NFs::Exists(CACHE_DIR + "roles"));
- {
- TFileInput f(CACHE_DIR + "roles");
- TString body = f.ReadAll();
- UNIT_ASSERT_C(body.Contains(ROLES), "got body: '" << body << "'");
- }
-
- fetchResult.Code = 304;
- fetchResult.Response.clear();
- UNIT_ASSERT_NO_EXCEPTION(fetcher.Update(NUtils::TFetchResult(fetchResult)));
- UNIT_ASSERT(fetcher.AreRolesOk());
- UNIT_ASSERT(NFs::Exists(CACHE_DIR + "roles"));
-
- fetchResult.Code = 200;
- fetchResult.Headers.AddHeader("X-Tirole-Compression", "kek");
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- fetcher.Update(NUtils::TFetchResult(fetchResult)),
- yexception,
- "unknown codec format version; known: 1; got: kek");
- }
-
- Y_UNIT_TEST(CreateTiroleRequest) {
- CleanCache();
- auto logger = MakeIntrusive<TLogger>();
-
- TRolesFetcherSettings s;
- s.CacheDir = CACHE_DIR;
- s.SelfTvmId = 111111;
- s.IdmSystemSlug = "some sys";
- TRolesFetcher fetcher(s, logger);
-
- TRolesFetcher::TRequest req = fetcher.CreateTiroleRequest("some_ticket");
- UNIT_ASSERT_VALUES_EQUAL(
- "/v1/get_actual_roles?system_slug=some+sys&_pid=&lib_version=client_",
- TStringBuf(req.Url).Chop(5));
- UNIT_ASSERT_VALUES_EQUAL(
- TKeepAliveHttpClient::THeaders({
- {"X-Ya-Service-Ticket", "some_ticket"},
- }),
- req.Headers);
-
- TDiskWriter wr(CACHE_DIR + "roles");
- UNIT_ASSERT(wr.Write(TRolesFetcher::PrepareDiskFormat(
- R"({"revision": "asd&qwe", "born_date": 42})",
- "some sys")));
- UNIT_ASSERT_NO_EXCEPTION(fetcher.ReadFromDisk());
-
- req = fetcher.CreateTiroleRequest("some_ticket");
- UNIT_ASSERT_VALUES_EQUAL(
- "/v1/get_actual_roles?system_slug=some+sys&_pid=&lib_version=client_",
- TStringBuf(req.Url).Chop(5));
- UNIT_ASSERT_VALUES_EQUAL(
- TKeepAliveHttpClient::THeaders({
- {"If-None-Match", R"("asd&qwe")"},
- {"X-Ya-Service-Ticket", "some_ticket"},
- }),
- req.Headers);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/roles/tvmtool_roles_fetcher_ut.cpp b/library/cpp/tvmauth/client/ut/roles/tvmtool_roles_fetcher_ut.cpp
deleted file mode 100644
index 55db4950ce..0000000000
--- a/library/cpp/tvmauth/client/ut/roles/tvmtool_roles_fetcher_ut.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-#include <library/cpp/tvmauth/client/ut/common.h>
-
-#include <library/cpp/tvmauth/client/misc/tool/roles_fetcher.h>
-
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NTvmTool;
-
-Y_UNIT_TEST_SUITE(TvmToolRolesFetcher) {
- static const TString ROLES = R"({"revision": "100501", "born_date": 42})";
-
- Y_UNIT_TEST(IsTimeToUpdate) {
- TRolesFetcher rf(
- TRolesFetcherSettings{.UpdatePeriod = TDuration::Minutes(1)},
- new TLogger);
-
- UNIT_ASSERT(!rf.IsTimeToUpdate(TDuration::Seconds(3)));
- UNIT_ASSERT(!rf.IsTimeToUpdate(TDuration::Seconds(60)));
- UNIT_ASSERT(rf.IsTimeToUpdate(TDuration::Seconds(61)));
- UNIT_ASSERT(rf.IsTimeToUpdate(TDuration::Seconds(600)));
- }
-
- Y_UNIT_TEST(ShouldWarn) {
- TRolesFetcher rf(
- TRolesFetcherSettings{.WarnPeriod = TDuration::Minutes(20)},
- new TLogger);
-
- UNIT_ASSERT(!rf.ShouldWarn(TDuration::Minutes(3)));
- UNIT_ASSERT(!rf.ShouldWarn(TDuration::Minutes(20)));
- UNIT_ASSERT(rf.ShouldWarn(TDuration::Minutes(21)));
- UNIT_ASSERT(rf.ShouldWarn(TDuration::Minutes(600)));
- }
-
- Y_UNIT_TEST(Common) {
- auto logger = MakeIntrusive<TLogger>();
- TRolesFetcher rf(
- TRolesFetcherSettings{.SelfAlias = "some_alias"},
- logger);
- UNIT_ASSERT(!rf.AreRolesOk());
- UNIT_ASSERT(!rf.GetCurrentRoles());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- rf.Update(NUtils::TFetchResult{.Code = HTTP_NOT_MODIFIED}),
- yexception,
- "tvmtool did not return any roles because current roles are actual, but there are no roles in memory - this should never happen");
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- rf.Update(NUtils::TFetchResult{.Code = HTTP_BAD_REQUEST, .Response = "kek"}),
- yexception,
- "Unexpected code from tvmtool: 400. kek");
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- rf.Update(NUtils::TFetchResult{.Code = HTTP_OK, .Response = "kek"}),
- yexception,
- "Invalid json. 'kek'");
-
- UNIT_ASSERT_NO_EXCEPTION(rf.Update(NUtils::TFetchResult{.Code = HTTP_OK, .Response = ROLES}));
- UNIT_ASSERT(rf.AreRolesOk());
- UNIT_ASSERT(rf.GetCurrentRoles());
- UNIT_ASSERT_VALUES_EQUAL("100501", rf.GetCurrentRoles()->GetMeta().Revision);
-
- UNIT_ASSERT_NO_EXCEPTION(rf.Update(NUtils::TFetchResult{.Code = HTTP_NOT_MODIFIED}));
- UNIT_ASSERT_VALUES_EQUAL("100501", rf.GetCurrentRoles()->GetMeta().Revision);
-
- UNIT_ASSERT_VALUES_EQUAL(
- "7: Succeed to update roles with revision 100501\n",
- logger->Stream.Str());
- }
-
- Y_UNIT_TEST(CreateRequest) {
- struct TTestFetcher: TRolesFetcher {
- using TRolesFetcher::CreateRequest;
- using TRolesFetcher::TRequest;
- using TRolesFetcher::TRolesFetcher;
- };
-
- TTestFetcher rf(
- TRolesFetcherSettings{.SelfAlias = "some_&alias"},
- new TLogger);
-
- TTestFetcher::TRequest request = rf.CreateRequest({{"some_header", "some_value"}});
- UNIT_ASSERT_VALUES_EQUAL(
- "/v2/roles?self=some_%26alias",
- request.Url);
- UNIT_ASSERT_VALUES_EQUAL(
- TKeepAliveHttpClient::THeaders({{"some_header", "some_value"}}),
- request.Headers);
-
- UNIT_ASSERT_NO_EXCEPTION(rf.Update(NUtils::TFetchResult{.Code = HTTP_OK, .Response = ROLES}));
-
- request = rf.CreateRequest({{"some_header", "some_value"}});
- UNIT_ASSERT_VALUES_EQUAL(
- "/v2/roles?self=some_%26alias",
- request.Url);
- UNIT_ASSERT_VALUES_EQUAL(
- TKeepAliveHttpClient::THeaders({
- {"some_header", "some_value"},
- {"If-None-Match", R"("100501")"},
- }),
- request.Headers);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/settings_ut.cpp b/library/cpp/tvmauth/client/ut/settings_ut.cpp
deleted file mode 100644
index 76c9542442..0000000000
--- a/library/cpp/tvmauth/client/ut/settings_ut.cpp
+++ /dev/null
@@ -1,169 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/misc/api/settings.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ClientSettings) {
-#if !defined(_win_)
- Y_UNIT_TEST(CheckValid) {
- struct TTestCase {
- TString Name;
- NTvmApi::TClientSettings Settings;
- TString Err;
- };
- std::vector<TTestCase> cases = {
- TTestCase{
- .Name = "default",
- .Settings = {},
- .Err = "Invalid settings: nothing to do",
- },
- TTestCase{
- .Name = "only secret",
- .Settings = {
- .Secret = TStringBuf("foobar"),
- },
- .Err = "Secret is present but destinations list is empty. It makes no sense",
- },
- TTestCase{
- .Name = "only dsts",
- .Settings = {
- .FetchServiceTicketsForDsts = {42},
- },
- .Err = "SelfTvmId cannot be 0 if fetching of Service Tickets required",
- },
- TTestCase{
- .Name = "dsts with selfTvmId",
- .Settings = {
- .SelfTvmId = 43,
- .FetchServiceTicketsForDsts = {42},
- },
- .Err = "Secret is required for fetching of Service Tickets",
- },
- TTestCase{
- .Name = "correct service tickets fetching",
- .Settings = {
- .SelfTvmId = 43,
- .Secret = TStringBuf("foobar"),
- .FetchServiceTicketsForDsts = {42},
- },
- .Err = "",
- },
- TTestCase{
- .Name = "only check srv flag",
- .Settings = {
- .CheckServiceTickets = true,
- },
- .Err = "SelfTvmId cannot be 0 if checking of Service Tickets required",
- },
- TTestCase{
- .Name = "tirole without disk cache",
- .Settings = {
- .SelfTvmId = 43,
- .Secret = TStringBuf("foobar"),
- .FetchRolesForIdmSystemSlug = "kek",
- },
- .Err = "Disk cache must be enabled to use roles: they can be heavy",
- },
- };
-
- for (const TTestCase& c : cases) {
- if (c.Err) {
- UNIT_ASSERT_EXCEPTION_CONTAINS_C(
- c.Settings.CheckValid(),
- TBrokenTvmClientSettings,
- c.Err,
- c.Name);
- } else {
- UNIT_ASSERT_NO_EXCEPTION_C(c.Settings.CheckValid(), c.Name);
- }
- }
-
- NTvmApi::TClientSettings s{.DiskCacheDir = "/impossible/dir"};
- UNIT_ASSERT_EXCEPTION(s.CheckValid(), TPermissionDenied);
- }
-
- Y_UNIT_TEST(CloneNormalized) {
- NTvmApi::TClientSettings original;
- original.FetchServiceTicketsForDsts = {43};
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(original.CloneNormalized(),
- TBrokenTvmClientSettings,
- "SelfTvmId cannot be 0 if fetching of Service Tickets required");
- original.SelfTvmId = 15;
- original.Secret = "bar";
- original.DiskCacheDir = "./";
-
- NTvmApi::TClientSettings::TDstVector expected = {43};
- UNIT_ASSERT_VALUES_EQUAL(expected, original.CloneNormalized().FetchServiceTicketsForDsts);
-
- original.FetchServiceTicketsForDstsWithAliases = {{"foo", 42}};
- expected = {42, 43};
- UNIT_ASSERT_VALUES_EQUAL(expected, original.CloneNormalized().FetchServiceTicketsForDsts);
-
- original.FetchRolesForIdmSystemSlug = "kek";
- expected = {42, 43, 2028120};
- UNIT_ASSERT_VALUES_EQUAL(expected, original.CloneNormalized().FetchServiceTicketsForDsts);
-
- original.FetchServiceTicketsForDsts.push_back(2028120);
- expected = {42, 43, 2028120};
- UNIT_ASSERT_VALUES_EQUAL(expected, original.CloneNormalized().FetchServiceTicketsForDsts);
- }
-
- Y_UNIT_TEST(NeedServiceTicketsFetching) {
- NTvmApi::TClientSettings s;
-
- UNIT_ASSERT(!s.NeedServiceTicketsFetching());
-
- s.FetchServiceTicketsForDsts = {42};
- UNIT_ASSERT(s.NeedServiceTicketsFetching());
- s.FetchServiceTicketsForDsts.clear();
-
- s.FetchServiceTicketsForDstsWithAliases = {{"foo", 42}};
- UNIT_ASSERT(s.NeedServiceTicketsFetching());
- s.FetchServiceTicketsForDstsWithAliases.clear();
-
- s.FetchRolesForIdmSystemSlug = "bar";
- UNIT_ASSERT(s.NeedServiceTicketsFetching());
- s.FetchRolesForIdmSystemSlug.clear();
- }
-
- Y_UNIT_TEST(permitions) {
- UNIT_ASSERT_EXCEPTION(NTvmApi::TClientSettings::CheckPermissions("/qwerty"), TPermissionDenied);
-
- const TString tmpDir = "./cache_dir";
-
- NFs::RemoveRecursive(tmpDir);
- NFs::MakeDirectory(tmpDir, NFs::FP_OWNER_WRITE | NFs::FP_GROUP_WRITE | NFs::FP_ALL_WRITE);
- UNIT_ASSERT_EXCEPTION(NTvmApi::TClientSettings::CheckPermissions(tmpDir), TPermissionDenied);
-
- NFs::RemoveRecursive(tmpDir);
- NFs::MakeDirectory(tmpDir, NFs::FP_OWNER_READ | NFs::FP_GROUP_READ | NFs::FP_ALL_READ);
- UNIT_ASSERT_EXCEPTION(NTvmApi::TClientSettings::CheckPermissions(tmpDir), TPermissionDenied);
-
- NFs::RemoveRecursive(tmpDir);
- NFs::MakeDirectory(tmpDir, NFs::FP_COMMON_FILE);
- UNIT_ASSERT_NO_EXCEPTION(NTvmApi::TClientSettings::CheckPermissions(tmpDir));
- }
-#endif
-
- Y_UNIT_TEST(Dst) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(NTvmApi::TClientSettings::TDst(0), yexception, "TvmId cannot be 0");
- UNIT_ASSERT_EXCEPTION_CONTAINS(NTvmApi::TClientSettings::TDstMap({{"blackbox", 0}}),
- TBrokenTvmClientSettings,
- "TvmId cannot be 0");
- }
-
- Y_UNIT_TEST(Fetching) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(125);
-
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- UNIT_ASSERT_NO_EXCEPTION(s.CheckValid());
-
- UNIT_ASSERT_VALUES_EQUAL(s.FetchServiceTicketsForDsts.size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(s.FetchServiceTicketsForDsts[0], 19);
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/src_checker_ut.cpp b/library/cpp/tvmauth/client/ut/src_checker_ut.cpp
deleted file mode 100644
index bd1646d6b9..0000000000
--- a/library/cpp/tvmauth/client/ut/src_checker_ut.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/mocked_updater.h>
-#include <library/cpp/tvmauth/client/misc/src_checker.h>
-#include <library/cpp/tvmauth/client/misc/api/threaded_updater.h>
-
-#include <library/cpp/tvmauth/type.h>
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(SrcChecker) {
- Y_UNIT_TEST(Ctor) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- TSrcChecker(new TMockedUpdater),
- TBrokenTvmClientSettings,
- "Need to use TClientSettings::EnableRolesFetching");
- }
-
- Y_UNIT_TEST(Check) {
- NRoles::TRolesPtr roles = std::make_shared<NRoles::TRoles>(
- NRoles::TRoles::TMeta{},
- NRoles::TRoles::TTvmConsumers{
- {12345, std::make_shared<NRoles::TConsumerRoles>(
- THashMap<TString, NRoles::TEntitiesPtr>())},
- },
- NRoles::TRoles::TUserConsumers{},
- std::make_shared<TString>());
- const TSrcChecker checker(new TMockedUpdater({.Roles = roles}));
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(
- checker.Check(NUnittest::CreateServiceTicket(ETicketStatus::Expired, 12345)),
- TIllegalUsage,
- "Service ticket must be valid");
-
- TCheckedServiceTicket ticket;
- UNIT_ASSERT_NO_EXCEPTION(
- ticket = checker.Check(NUnittest::CreateServiceTicket(ETicketStatus::Ok, 12345)));
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, ticket.GetStatus());
-
- UNIT_ASSERT_NO_EXCEPTION(
- ticket = checker.Check(NUnittest::CreateServiceTicket(ETicketStatus::Ok, 9999)));
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::NoRoles, ticket.GetStatus());
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/tvmapi_updater_ut.cpp b/library/cpp/tvmauth/client/ut/tvmapi_updater_ut.cpp
deleted file mode 100644
index 4cf449711b..0000000000
--- a/library/cpp/tvmauth/client/ut/tvmapi_updater_ut.cpp
+++ /dev/null
@@ -1,1272 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/mocked_updater.h>
-#include <library/cpp/tvmauth/client/misc/disk_cache.h>
-#include <library/cpp/tvmauth/client/misc/api/threaded_updater.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/stream/file.h>
-#include <util/string/subst.h>
-#include <util/system/fs.h>
-
-#include <regex>
-
-using namespace NTvmAuth;
-static const std::regex TIME_REGEX(R"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d{6}Z)");
-
-Y_UNIT_TEST_SUITE(ApiUpdater) {
- static const TString SRV_TICKET = "3:serv:CBAQ__________9_IgYIexCUkQY:GioCM49Ob6_f80y6FY0XBVN4hLXuMlFeyMvIMiDuQnZkbkLpRpQOuQo5YjWoBjM0Vf-XqOm8B7xtrvxSYHDD7Q4OatN2l-Iwg7i71lE3scUeD36x47st3nd0OThvtjrFx_D8mw_c0GT5KcniZlqq1SjhLyAk1b_zJsx8viRAhCU";
- static const TString TEST_TICKET = "3:user:CA0Q__________9_Gg4KAgh7EHsg0oXYzAQoAQ:FSADps3wNGm92Vyb1E9IVq5M6ZygdGdt1vafWWEhfDDeCLoVA-sJesxMl2pGW4OxJ8J1r_MfpG3ZoBk8rLVMHUFrPa6HheTbeXFAWl8quEniauXvKQe4VyrpA1SPgtRoFqi5upSDIJzEAe1YRJjq1EClQ_slMt8R0kA_JjKUX54";
- static const TString TVM_RESPONSE =
- R"({
- "19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- "213" : { "ticket" : "service_ticket_2"},
- "234" : { "error" : "Dst is not found" },
- "185" : { "ticket" : "service_ticket_3"},
- "deprecated" : { "ticket" : "deprecated_ticket" }
- })";
-
- static const TString CACHE_DIR = "./tmp/";
-
- static void CleanCache() {
- NFs::RemoveRecursive(CACHE_DIR);
- NFs::MakeDirectoryRecursive(CACHE_DIR);
- }
-
- Y_UNIT_TEST(MockedUpdater) {
- TMockedUpdater m;
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, m.GetStatus());
- UNIT_ASSERT(m.GetCachedServiceContext()->Check(SRV_TICKET));
- UNIT_ASSERT(m.GetCachedUserContext()->Check(TEST_TICKET));
- }
-
- Y_UNIT_TEST(Updater) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(GetCachePath());
-
- auto l = MakeIntrusive<TLogger>();
- {
- auto u = NTvmApi::TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus());
- }
-
- UNIT_ASSERT_C(l->Stream.Str().find("was successfully read") != TString::npos, l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().find("were successfully fetched") == TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(Updater_badConfig) {
- NTvmApi::TClientSettings s;
- UNIT_ASSERT_EXCEPTION(NTvmApi::TThreadedUpdater::Create(s, TDevNullLogger::IAmBrave()), yexception);
- s.SetSelfTvmId(100500);
- UNIT_ASSERT_EXCEPTION(NTvmApi::TThreadedUpdater::Create(s, TDevNullLogger::IAmBrave()), yexception);
- s.SetDiskCacheDir(GetCachePath());
- UNIT_ASSERT_EXCEPTION(NTvmApi::TThreadedUpdater::Create(s, TDevNullLogger::IAmBrave()), yexception);
- }
-
- class TOfflineUpdater: public NTvmApi::TThreadedUpdater {
- bool Enabled_;
- TString PublicKeys_;
-
- public:
- TOfflineUpdater(const NTvmApi::TClientSettings& settings,
- TIntrusivePtr<TLogger> l,
- bool enabled = false,
- TString keys = NUnittest::TVMKNIFE_PUBLIC_KEYS)
- : NTvmApi::TThreadedUpdater(settings, l)
- , Enabled_(enabled)
- , PublicKeys_(keys)
- {
- Init();
- StartWorker();
- }
-
- NUtils::TFetchResult FetchServiceTicketsFromHttp(const TString&) const override {
- if (!Enabled_) {
- throw yexception() << "alarm";
- }
- return {200, {}, "/2/ticket", TVM_RESPONSE, ""};
- }
-
- NUtils::TFetchResult FetchPublicKeysFromHttp() const override {
- if (!Enabled_) {
- throw yexception() << "alarm";
- }
- return {200, {}, "/2/keys", PublicKeys_, ""};
- }
- };
-
- Y_UNIT_TEST(StartWithoutCache) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.EnableServiceTicketChecking();
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry:");
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n"
- << "4: Failed to get ServiceTickets: alarm\n"
- << "4: Failed to get ServiceTickets: alarm\n"
- << "4: Failed to get ServiceTickets: alarm\n"
- << "4: Failed to update service tickets: alarm\n"
- << "3: Service tickets have not been refreshed for too long period\n",
- l->Stream.Str());
- }
-
- static void WriteFile(TString name, TStringBuf body, TInstant time) {
- NFs::Remove(CACHE_DIR + name);
- TFileOutput f(CACHE_DIR + name);
- f << TDiskWriter::PrepareData(time, body);
- }
-
- Y_UNIT_TEST(StartWithOldCache) {
- CleanCache();
- WriteFile("./public_keys",
- NUnittest::TVMKNIFE_PUBLIC_KEYS,
- TInstant::Now() - TDuration::Days(30)); // too old
- WriteFile("./service_tickets",
- R"({"19":{"ticket":"3:serv:CBAQACIGCJSRBhAL:Fi"}})"
- "\t100500",
- TInstant::Now()); // too old
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_C(l->Stream.Str().find("Disk cache (public keys) is too old") != TString::npos, l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().find("Disk cache (service tickets) is too old") != TString::npos, l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().find("were successfully fetched") != TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithMissingCache) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir("../");
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry: ");
-
- UNIT_ASSERT_C(l->Stream.Str().find("does not exist") != TString::npos, l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().find("were successfully fetched") == TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithBadCache_Tickets) {
- CleanCache();
- WriteFile("./service_tickets",
- TVM_RESPONSE,
- TInstant::Now());
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "4: Failed to read service tickets from disk: YYYYYYYYYYYYYYY\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"),
- std::regex(R"(Failed to read service tickets from disk: [^\n]+)"),
- "Failed to read service tickets from disk: YYYYYYYYYYYYYYY"));
- }
-
- Y_UNIT_TEST(StartWithBadCache_PublicKeys) {
- CleanCache();
- WriteFile("./public_keys",
- "ksjdafnlskdjzfgbhdl",
- TInstant::Now());
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry:");
-
- UNIT_ASSERT_C(l->Stream.Str().find("4: Failed to read public keys from disk: Malformed TVM keys") != TString::npos, l->Stream.Str());
- }
-
- Y_UNIT_TEST(StartWithCacheForAnotherTvmId) {
- CleanCache();
- WriteFile("./service_tickets",
- TVM_RESPONSE + "\t" + "100499",
- TInstant::Now());
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "4: Disk cache is for another tvmId (100499). Self=100500\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(StartWithCacheForAnotherDsts) {
- CleanCache();
- TInstant now = TInstant::Now();
- WriteFile("./service_tickets",
- R"({"213" : { "ticket" : "3:serv:CBAQ__________9_IgYIlJEGEAs:T-"}})"
- "\t"
- "100500",
- now);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- auto cache = u.GetCachedServiceTickets();
- UNIT_ASSERT(cache->TicketsById.contains(213));
- UNIT_ASSERT(cache->TicketsById.contains(19));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): " << TInstant::Seconds(now.Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 1 destination(s)\n"
- << "6: Cache was partly updated with 1 service ticket(s). total: 2\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- l->Stream.Clear();
-
- {
- TOfflineUpdater u(s, l, true);
- auto cache = u.GetCachedServiceTickets();
- UNIT_ASSERT(cache->TicketsById.contains(213));
- UNIT_ASSERT(cache->TicketsById.contains(19));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 2 service ticket(s) from disk\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(StartWithNotFreshCacheForAnotherDsts) {
- CleanCache();
- TInstant now = TInstant::Now();
- WriteFile("./service_tickets",
- R"({"213" : { "ticket" : "3:serv:CBAQ__________9_IgYIlJEGEAs:T-"}})"
- "\t"
- "100500",
- now - TDuration::Hours(2));
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- auto cache = u.GetCachedServiceTickets();
- UNIT_ASSERT(cache->TicketsById.contains(213));
- UNIT_ASSERT(cache->TicketsById.contains(19));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- l->Stream.Clear();
-
- {
- TOfflineUpdater u(s, l, true);
- auto cache = u.GetCachedServiceTickets();
- UNIT_ASSERT(cache->TicketsById.contains(213));
- UNIT_ASSERT(cache->TicketsById.contains(19));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 2 service ticket(s) from disk\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(StartWithPartialDiskCache) {
- CleanCache();
- WriteFile("./public_keys",
- NUnittest::TVMKNIFE_PUBLIC_KEYS,
- TInstant::Now());
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"kolmo", 213}});
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: File './tmp/service_tickets' does not exist\n"
- << "6: File './tmp/public_keys' was successfully read\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "6: Cache was updated with 2 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(StartFromHttpAndRestartFromDisk) {
- CleanCache();
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
- s.SetDiskCacheDir(CACHE_DIR);
-
- {
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: File './tmp/service_tickets' does not exist\n"
- << "7: File './tmp/public_keys' does not exist\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 1 destination(s)\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/service_tickets' was successfully written\n"
- << "7: Public keys were successfully fetched from https://tvm-api.yandex.net\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "6: File './tmp/public_keys' was successfully written\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- {
- auto l = MakeIntrusive<TLogger>();
- {
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/service_tickets' was successfully read\n"
- << "6: Got 1 service ticket(s) from disk\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n"
- << "6: File './tmp/public_keys' was successfully read\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
- }
-
- class TUnstableUpdater: public NTvmApi::TThreadedUpdater {
- mutable int V1_ = 0;
- mutable int V2_ = 0;
-
- public:
- TUnstableUpdater(const NTvmApi::TClientSettings& settings, TIntrusivePtr<TLogger> l)
- : NTvmApi::TThreadedUpdater(settings, l)
- {
- UNIT_ASSERT_NO_EXCEPTION_C(Init(), l->Stream.Str());
- ExpBackoff_.SetEnabled(false);
- StartWorker();
-
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Ok, GetStatus(), l->Stream.Str());
-
- Sleep(TDuration::MicroSeconds(100));
- PublicKeysDurations_.Expiring = TDuration::MicroSeconds(100);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus(TClientStatus::Warning, "Internal client error: failed to collect last useful error message, please report this message to tvm-dev@yandex-team.ru"),
- GetStatus(),
- l->Stream.Str());
-
- PublicKeysDurations_.Invalid = TDuration::MicroSeconds(20);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Error, GetStatus(), l->Stream.Str());
-
- PublicKeysDurations_.Expiring = TDuration::Seconds(100);
- PublicKeysDurations_.Invalid = TDuration::Seconds(200);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Ok, GetStatus(), l->Stream.Str());
-
- ServiceTicketsDurations_.Expiring = TDuration::MicroSeconds(100);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Warning, GetStatus(), l->Stream.Str());
-
- ServiceTicketsDurations_.Invalid = TDuration::MicroSeconds(20);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Warning, GetStatus(), l->Stream.Str());
-
- const TInstant* inv = &GetCachedServiceTickets()->InvalidationTime;
- *const_cast<TInstant*>(inv) = TInstant::Now() + TDuration::Seconds(30);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Error, GetStatus(), l->Stream.Str());
- }
-
- NUtils::TFetchResult FetchServiceTicketsFromHttp(const TString&) const override {
- Y_ENSURE_EX(++V1_ > 1, yexception() << "++v1_ > 1:" << V1_);
- return {200, {}, "/2/ticket", TVM_RESPONSE, ""};
- }
-
- NUtils::TFetchResult FetchPublicKeysFromHttp() const override {
- Y_ENSURE_EX(++V2_ > 2, yexception() << "++v2_ > 2:" << V2_);
- return {200, {}, "/2/keys", NUnittest::TVMKNIFE_PUBLIC_KEYS, ""};
- }
- };
-
- Y_UNIT_TEST(StartFromUnstableHttp) {
- CleanCache();
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TUnstableUpdater u(s, l);
- }
-
- UNIT_ASSERT_C(l->Stream.Str().Contains("++v1_ > 1"), l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().Contains("++v2_ > 2"), l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().Contains("7: Response with service tickets for 1 destination(s) was successfully fetched from https://tvm-api.yandex.net"), l->Stream.Str());
- UNIT_ASSERT_C(l->Stream.Str().Contains("7: Public keys were successfully fetched"), l->Stream.Str());
- }
-
- Y_UNIT_TEST(GetUpdateTimeOfServiceTickets) {
- CleanCache();
- TInstant ins = TInstant::Now();
- WriteFile("./service_tickets",
- TVM_RESPONSE + "\t" + "100500",
- ins);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- TOfflineUpdater u(s, l, true);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), u.GetUpdateTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(ins.Seconds()), u.GetUpdateTimeOfServiceTickets());
- }
-
- class TSignalingUpdater: public NTvmApi::TThreadedUpdater {
- mutable int V_ = 0;
- TAutoEvent& Ev_;
- const TStringBuf PublicKeys_;
-
- public:
- TSignalingUpdater(const NTvmApi::TClientSettings& settings,
- TLoggerPtr l,
- TAutoEvent& ev,
- const TStringBuf keys = NUnittest::TVMKNIFE_PUBLIC_KEYS)
- : NTvmApi::TThreadedUpdater(settings, l)
- , Ev_(ev)
- , PublicKeys_(keys)
- {
- WorkerAwakingPeriod_ = TDuration::MilliSeconds(300);
- PublicKeysDurations_.RefreshPeriod = TDuration::MilliSeconds(700);
- Init();
- ExpBackoff_.SetEnabled(false);
- StartWorker();
- }
-
- NUtils::TFetchResult FetchPublicKeysFromHttp() const override {
- if (++V_ >= 2) {
- Ev_.Signal();
- }
- return {200, {}, "/2/keys", TString(PublicKeys_), ""};
- }
- };
-
- Y_UNIT_TEST(StartWorker) {
- class TSignalingUpdater: public NTvmApi::TThreadedUpdater {
- mutable int V_ = 0;
- TAutoEvent& Ev_;
-
- public:
- TSignalingUpdater(const NTvmApi::TClientSettings& settings, TLoggerPtr l, TAutoEvent& ev)
- : NTvmApi::TThreadedUpdater(settings, l)
- , Ev_(ev)
- {
- WorkerAwakingPeriod_ = TDuration::MilliSeconds(300);
- PublicKeysDurations_.RefreshPeriod = TDuration::MilliSeconds(700);
- Init();
- ExpBackoff_.SetEnabled(false);
- StartWorker();
- }
-
- void Worker() override {
- NTvmApi::TThreadedUpdater::Worker();
- Ev_.Signal();
- }
-
- NUtils::TFetchResult FetchPublicKeysFromHttp() const override {
- if (++V_ < 4) {
- return {500, {}, "/2/keys", "lol", ""};
- }
- return {200, {}, "/2/keys", NUnittest::TVMKNIFE_PUBLIC_KEYS, "CAEQChkAAAAAAAD4PyGamZmZmZm5PyhkMAE4B0BGSAI"};
- }
- };
-
- CleanCache();
- TInstant expiringPubKeys = TInstant::Now() - TDuration::Days(3);
- WriteFile("./public_keys", NUnittest::TVMKNIFE_PUBLIC_KEYS, expiringPubKeys);
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- TAutoEvent ev;
- {
- TSignalingUpdater u(s, l, ev);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus(TClientStatus::Warning, "PublicKeys: Path:/2/keys.Code=500: lol"),
- u.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(expiringPubKeys.Seconds()), u.GetUpdateTimeOfPublicKeys());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), u.GetUpdateTimeOfServiceTickets());
-
- UNIT_ASSERT(ev.WaitT(TDuration::Seconds(15)));
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/public_keys' was successfully read\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "4: Failed to get PublicKeys: Path:/2/keys.Code=500: lol\n"
- << "4: Failed to get PublicKeys: Path:/2/keys.Code=500: lol\n"
- << "4: Failed to get PublicKeys: Path:/2/keys.Code=500: lol\n"
- << "4: Failed to update public keys: Path:/2/keys.Code=500: lol\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "7: Thread-worker started\n"
- << "7: Retry settings were updated: exponential_backoff_min:0.000000s->1.000000s;exponential_backoff_max:60.000000s->10.000000s;exponential_backoff_factor:2->1.5;exponential_backoff_jitter:0.5->0.1;max_random_sleep_default:5.000000s->0.100000s;retries_on_start:3->1;worker_awaking_period:10.000000s->7.000000s;dsts_limit:300->70;\n"
- << "6: File './tmp/retry_settings' was successfully written\n"
- << "7: Public keys were successfully fetched from https://tvm-api.yandex.net\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "6: File './tmp/public_keys' was successfully written\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
-#if defined(_unix_)
- Y_UNIT_TEST(StartFromCacheAndBadPublicKeysFromHttp) {
- CleanCache();
- TInstant now = TInstant::Now();
- WriteFile("public_keys", NUnittest::TVMKNIFE_PUBLIC_KEYS, now - TDuration::Days(3)); // expiring public keys
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.SetDiskCacheDir(CACHE_DIR);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TAutoEvent ev;
- TSignalingUpdater u(s, l, ev, "malformed keys");
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus(TClientStatus::Warning, "PublicKeys: Malformed TVM keys"),
- u.GetStatus());
-
- UNIT_ASSERT(ev.WaitT(TDuration::Seconds(15)));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, u.GetStatus());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: File './tmp/public_keys' was successfully read\n"
- << "6: Cache was updated with public keys: " << TInstant::Seconds((now - TDuration::Days(3)).Seconds()) << "\n"
- << "7: File './tmp/retry_settings' does not exist\n"
- << "7: Public keys were successfully fetched from https://tvm-api.yandex.net\n"
- << "4: Failed to update public keys: Malformed TVM keys\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "7: Thread-worker started\n"
- << "7: Public keys were successfully fetched from https://tvm-api.yandex.net\n"
- << "4: Failed to update public keys: Malformed TVM keys\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- }
-#endif
-
- Y_UNIT_TEST(StartWithBadPublicKeysFromHttp) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
-
- auto l = MakeIntrusive<TLogger>();
- TAutoEvent ev;
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l, true, "some public keys"),
- TRetriableException,
- "Failed to start TvmClient. You can retry:");
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n"
- << "7: Public keys were successfully fetched from https://tvm-api.yandex.net\n"
- << "4: Failed to update public keys: Malformed TVM keys\n"
- << "3: Public keys have not been refreshed for too long period\n",
- l->Stream.Str());
- }
-
- class TNotInitedUpdater: public NTvmApi::TThreadedUpdater {
- public:
- TNotInitedUpdater(const NTvmApi::TClientSettings& settings, TLoggerPtr l = TDevNullLogger::IAmBrave())
- : NTvmApi::TThreadedUpdater(settings, l)
- {
- this->ExpBackoff_.SetEnabled(false);
- }
-
- using NTvmApi::TThreadedUpdater::AppendToJsonArray;
- using NTvmApi::TThreadedUpdater::AreServicesTicketsOk;
- using NTvmApi::TThreadedUpdater::CreateJsonArray;
- using NTvmApi::TThreadedUpdater::FindMissingDsts;
- using NTvmApi::TThreadedUpdater::GetPublicKeysFromHttp;
- using NTvmApi::TThreadedUpdater::GetServiceTicketsFromHttp;
- using NTvmApi::TThreadedUpdater::Init;
- using NTvmApi::TThreadedUpdater::IsServiceContextOk;
- using NTvmApi::TThreadedUpdater::IsTimeToUpdatePublicKeys;
- using NTvmApi::TThreadedUpdater::IsTimeToUpdateServiceTickets;
- using NTvmApi::TThreadedUpdater::IsUserContextOk;
- using NTvmApi::TThreadedUpdater::ParseTicketsFromDisk;
- using NTvmApi::TThreadedUpdater::ParseTicketsFromResponse;
- using NTvmApi::TThreadedUpdater::PrepareRequestForServiceTickets;
- using NTvmApi::TThreadedUpdater::PrepareTicketsForDisk;
- using NTvmApi::TThreadedUpdater::SetServiceContext;
- using NTvmApi::TThreadedUpdater::SetServiceTickets;
- using NTvmApi::TThreadedUpdater::SetUserContext;
- using NTvmApi::TThreadedUpdater::THttpResult;
- using NTvmApi::TThreadedUpdater::TPairTicketsErrors;
- using TAsyncUpdaterBase::IsServiceTicketMapOk;
- };
-
- Y_UNIT_TEST(IsCacheComplete_Empty) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"blackbox2", 20}});
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
-
- TNotInitedUpdater u(s);
- UNIT_ASSERT(!u.AreServicesTicketsOk());
- }
-
- Y_UNIT_TEST(IsCacheComplete_Tickets) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"blackbox2", 20}});
-
- TNotInitedUpdater u(s);
- UNIT_ASSERT(!u.AreServicesTicketsOk());
-
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- TServiceTickets::TMapIdStr({{1, "mega_ticket"}}),
- TServiceTickets::TMapIdStr({{2, "mega_error"}}),
- TServiceTickets::TMapAliasId()));
- UNIT_ASSERT(!u.AreServicesTicketsOk());
-
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- TServiceTickets::TMapIdStr({
- {1, "mega_ticket"},
- {2, "mega_ticket2"},
- }),
- TServiceTickets::TMapIdStr({
- {3, "mega_error3"},
- }),
- TServiceTickets::TMapAliasId()));
- UNIT_ASSERT(u.AreServicesTicketsOk());
- }
-
- Y_UNIT_TEST(IsCacheComplete_Service) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
-
- TNotInitedUpdater u(s);
- UNIT_ASSERT(!u.IsServiceContextOk());
-
- u.SetServiceContext(MakeIntrusiveConst<TServiceContext>(
- TServiceContext::CheckingFactory(100500, NUnittest::TVMKNIFE_PUBLIC_KEYS)));
- UNIT_ASSERT(u.IsServiceContextOk());
- }
-
- Y_UNIT_TEST(IsCacheComplete_User) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
-
- TNotInitedUpdater u(s);
- UNIT_ASSERT(!u.IsUserContextOk());
-
- u.SetUserContext(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT(u.IsUserContextOk());
- }
-
- Y_UNIT_TEST(TicketsOnDisk) {
- TString res = R"({
- "19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- "213" : { "ticket" : "service_ticket_2"},
- "234" : { "error" : "Dst is not found" },
- "185" : { "ticket" : "service_ticket_3"},
- "deprecated" : { "ticket" : "deprecated_ticket" }
- })";
- res.append("\t100500");
-
- UNIT_ASSERT_VALUES_EQUAL(res, TNotInitedUpdater::PrepareTicketsForDisk(TVM_RESPONSE, 100500));
-
- auto pair = TNotInitedUpdater::ParseTicketsFromDisk(res);
- UNIT_ASSERT_VALUES_EQUAL(pair.first, TVM_RESPONSE);
- UNIT_ASSERT_VALUES_EQUAL(pair.second, 100500);
-
- res.push_back('a');
- UNIT_ASSERT_EXCEPTION(TNotInitedUpdater::ParseTicketsFromDisk(res), yexception);
- }
-
- Y_UNIT_TEST(IsTimeToUpdatePublicKeys) {
- NTvmApi::TClientSettings s;
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
-
- TNotInitedUpdater u(s);
-
- UNIT_ASSERT(!u.IsTimeToUpdatePublicKeys(TInstant::Now()));
- UNIT_ASSERT(!u.IsTimeToUpdatePublicKeys(TInstant::Now() - TDuration::Hours(23)));
- UNIT_ASSERT(u.IsTimeToUpdatePublicKeys(TInstant::Now() - TDuration::Days(1) - TDuration::MilliSeconds(1)));
- }
-
- Y_UNIT_TEST(IsTimeToUpdateServiceTickets) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}, {"blackbox2", 20}});
-
- TNotInitedUpdater u(s);
-
- UNIT_ASSERT(!u.IsTimeToUpdateServiceTickets(TInstant::Now() - TDuration::Minutes(59)));
- UNIT_ASSERT(u.IsTimeToUpdateServiceTickets(TInstant::Now() - TDuration::Hours(1) - TDuration::MilliSeconds(1)));
- }
-
- Y_UNIT_TEST(StartWithIncompliteCache) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", NTvmApi::TClientSettings::TDstVector({19, 20}));
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TOfflineUpdater(s, l, true),
- TNonRetriableException,
- "Failed to get ServiceTicket for 20: Missing tvm_id in response, should never happend: 20");
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from https://tvm-api.yandex.net\n"
- << "7: Got responses with service tickets with 1 pages for 2 destination(s)\n"
- << "3: Failed to get service ticket for dst=20: Missing tvm_id in response, should never happend: 20\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(PrepareRequestForServiceTickets) {
- const TServiceContext ctx = TServiceContext::SigningFactory("AAAAAAAAAAAAAAAAAAAAAA");
-
- TString s = TNotInitedUpdater::PrepareRequestForServiceTickets(117,
- ctx,
- {19, 20},
- NUtils::TProcInfo{
- "__some_pid__",
- "__some_pname__",
- "kar",
- },
- 100700);
- SubstGlobal(s.resize(s.size() - 5), "deb_", "");
- UNIT_ASSERT_VALUES_EQUAL("grant_type=client_credentials&src=117&dst=19,20&ts=100700&sign=XTz2Obd6PII_BHxswzWPJTjju9SrKsN6hyu1VsyxBvU&get_retry_settings=yes&_pid=__some_pid__&_procces_name=__some_pname__&lib_version=client_kar",
- s);
-
- s = TNotInitedUpdater::PrepareRequestForServiceTickets(118,
- ctx,
- {19},
- NUtils::TProcInfo{
- "__some_pid__",
- {},
- "kva_",
- },
- 100900);
- SubstGlobal(s.resize(s.size() - 5), "deb_", "");
- UNIT_ASSERT_VALUES_EQUAL("grant_type=client_credentials&src=118&dst=19&ts=100900&sign=-trBo9AtBLjp2ihy6cFAdMAQ6S9afHj23rFzYQ32jkQ&get_retry_settings=yes&_pid=__some_pid__&lib_version=client_kva_",
- s);
- }
-
- Y_UNIT_TEST(ParseTicketsFromResponse) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
-
- auto l = MakeIntrusive<TLogger>();
- TNotInitedUpdater u(s, l);
-
- TNotInitedUpdater::TPairTicketsErrors t;
- UNIT_ASSERT_EXCEPTION_CONTAINS(u.ParseTicketsFromResponse("{", NTvmApi::TDstSet{19}, t),
- yexception,
- "Invalid json from tvm-api");
-
- t = {};
- u.ParseTicketsFromResponse(TVM_RESPONSE, NTvmApi::TDstSet{19}, t);
-
- TNotInitedUpdater::TPairTicketsErrors expected{{{19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}}, {}};
- UNIT_ASSERT_VALUES_EQUAL("6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n",
- l->Stream.Str());
- UNIT_ASSERT_EQUAL(expected, t);
-
- t = {};
- u.ParseTicketsFromResponse(TVM_RESPONSE,
- NTvmApi::TDstSet{19, 213, 234, 235},
- t);
- expected = {{{19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- {213, "service_ticket_2"}},
- {{234, "Dst is not found"},
- {235, "Missing tvm_id in response, should never happend: 235"}}};
- UNIT_ASSERT_EQUAL(expected, t);
- UNIT_ASSERT_VALUES_EQUAL("6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n",
- l->Stream.Str());
-
- t = {};
- u.ParseTicketsFromResponse(
- R"([
- {"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},"234" : { "error" : "Dst is not found" }},
- {"213" : { "ticket" : "service_ticket_2"},"185" : { "ticket" : "service_ticket_3"}},
- {"deprecated" : { "ticket" : "deprecated_ticket" }}
- ])",
- NTvmApi::TDstSet{19, 213, 234, 235},
- t);
- expected = {{{19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- {213, "service_ticket_2"}},
- {{234, "Dst is not found"},
- {235, "Missing tvm_id in response, should never happend: 235"}}};
- UNIT_ASSERT_EQUAL(expected, t);
- UNIT_ASSERT_VALUES_EQUAL("6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(ParseTicketsFromResponseAsArray) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
-
- auto l = MakeIntrusive<TLogger>();
- TNotInitedUpdater u(s, l);
-
- TNotInitedUpdater::TPairTicketsErrors t;
- UNIT_ASSERT_EXCEPTION_CONTAINS(u.ParseTicketsFromResponse("[", NTvmApi::TDstSet{19}, t),
- yexception,
- "Invalid json from tvm-api");
-
- u.ParseTicketsFromResponse(R"([])", NTvmApi::TDstSet{19}, t);
- UNIT_ASSERT_VALUES_EQUAL("6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n",
- l->Stream.Str());
- TNotInitedUpdater::TPairTicketsErrors expected = {
- {}, {{19, "Missing tvm_id in response, should never happend: 19"}}};
- UNIT_ASSERT_VALUES_EQUAL(expected, t);
- l->Stream.Clear();
-
- t = {};
- u.ParseTicketsFromResponse(
- R"([{},{"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}}, {"213" : { "ticket" : "service_ticket_2"}}])",
- NTvmApi::TDstSet{19},
- t);
- UNIT_ASSERT_VALUES_EQUAL("", l->Stream.Str());
- expected = {{{19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}}, {}};
- UNIT_ASSERT_EQUAL(expected, t);
-
- t = {};
- u.ParseTicketsFromResponse(
- R"([{
- "19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}
- },
- {
- "213" : { "ticket" : "service_ticket_2"},
- "234" : { "error" : "Dst is not found" }
- },
- {
- "185" : { "ticket" : "service_ticket_3"},
- "deprecated" : { "ticket" : "deprecated_ticket" }
- }
- ])",
- NTvmApi::TDstSet{19, 213, 234, 235},
- t);
- expected = {{{19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- {213, "service_ticket_2"}},
- {{234, "Dst is not found"},
- {235, "Missing tvm_id in response, should never happend: 235"}}};
- UNIT_ASSERT_EQUAL(expected, t);
- UNIT_ASSERT_VALUES_EQUAL("", l->Stream.Str());
- }
-
- class TReplier: public TRequestReplier {
- public:
- HttpCodes Code = HTTP_OK;
-
- bool DoReply(const TReplyParams& params) override {
- TParsedHttpFull fl(params.Input.FirstLine());
-
- THttpResponse resp(Code);
- if (fl.Path == "/2/keys") {
- resp.SetContent(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- } else if (fl.Path == "/2/ticket") {
- resp.SetContent(TVM_RESPONSE);
- } else {
- UNIT_ASSERT(false);
- }
- resp.OutTo(params.Output);
-
- return true;
- }
- };
-
- class TOnlineUpdater: public NTvmApi::TThreadedUpdater {
- public:
- TOnlineUpdater(const NTvmApi::TClientSettings& settings, TIntrusivePtr<TLogger> l)
- : NTvmApi::TThreadedUpdater(settings, l)
- {
- Init();
- ExpBackoff_.SetEnabled(false);
- StartWorker();
- }
- };
-
- Y_UNIT_TEST(MocServerOk) {
- TPortManager pm;
- ui16 tvmPort = pm.GetPort(80);
- NMock::TMockServer server(tvmPort, []() { return new TReplier; });
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
- s.SetTvmHostPort("http://localhost", tvmPort);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TOnlineUpdater u(s, l);
- UNIT_ASSERT_VALUES_EQUAL_C(TClientStatus::Ok, u.GetStatus(), l->Stream.Str());
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from http://localhost\n"
- << "7: Got responses with service tickets with 1 pages for 1 destination(s)\n"
- << "6: Cache was updated with 1 service ticket(s): XXXXXXXXXXX\n"
- << "7: Public keys were successfully fetched from http://localhost\n"
- << "6: Cache was updated with public keys: XXXXXXXXXXX\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- std::regex_replace(std::string(l->Stream.Str()), TIME_REGEX, "XXXXXXXXXXX"));
- }
-
- Y_UNIT_TEST(MocServerBad) {
- TPortManager pm;
- ui16 tvmPort = pm.GetPort(80);
- NMock::TMockServer server(tvmPort,
- []() {
- auto p = new TReplier;
- p->Code = HTTP_BAD_REQUEST;
- return p;
- });
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", {{"blackbox", 19}});
- s.EnableServiceTicketChecking();
- s.EnableUserTicketChecking(EBlackboxEnv::Test);
- s.SetTvmHostPort("localhost", tvmPort);
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS_C(TOnlineUpdater(s, l),
- TNonRetriableException,
- "Failed to start TvmClient. Do not retry: ServiceTickets: Path:/2/ticket.Code=400:",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(MocServerPaginated) {
- class TReplier: public TRequestReplier {
- public:
- TString Response;
- TReplier(TString response)
- : Response(response)
- {
- }
-
- bool DoReply(const TReplyParams& params) override {
- TParsedHttpFull fl(params.Input.FirstLine());
- if (fl.Path != "/2/ticket") {
- UNIT_ASSERT_C(false, fl.Path);
- }
-
- THttpResponse resp(HTTP_OK);
- resp.SetContent(Response);
- resp.OutTo(params.Output);
- return true;
- }
- };
-
- TPortManager pm;
- ui16 tvmPort = pm.GetPort(80);
- TVector<TString> responses = {
- R"({"15" : { "ticket" : "service_ticket_3" },"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})",
- R"({"222" : { "ticket" : "service_ticket_2"}, "239" : { "error" : "Dst is not found" }})",
- R"({"185" : { "ticket" : "service_ticket_3"}})",
- };
- NMock::TMockServer server(tvmPort, [&responses]() {
- if (responses.empty()) {
- return new TReplier("<NULL>");
- }
- TString r = responses.front();
- responses.erase(responses.begin());
- return new TReplier(r);
- });
-
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketsFetchOptions("qwerty", NTvmApi::TClientSettings::TDstVector{19, 222, 239, 100500, 15});
- s.SetTvmHostPort("http://localhost", tvmPort);
-
- auto l = MakeIntrusive<TLogger>();
- {
- TNotInitedUpdater u(s, l);
- TNotInitedUpdater::THttpResult result = u.GetServiceTicketsFromHttp(NTvmApi::TDstSet{19, 222, 239, 100500, 15}, 2);
- UNIT_ASSERT_VALUES_EQUAL(TSmallVec<TString>({
- R"({"15" : { "ticket" : "service_ticket_3" },"19" : { "ticket" : "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}})",
- R"({"222" : { "ticket" : "service_ticket_2"}, "239" : { "error" : "Dst is not found" }})",
- R"({"185" : { "ticket" : "service_ticket_3"}})",
- }),
- result.Responses);
- TNotInitedUpdater::TPairTicketsErrors expected{
- {
- {19, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"},
- {222, "service_ticket_2"},
- {15, "service_ticket_3"},
- },
- {
- {239, "Dst is not found"},
- {100500, "Missing tvm_id in response, should never happend: 100500"},
- },
- };
- UNIT_ASSERT_VALUES_EQUAL(expected, result.TicketsWithErrors);
- }
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "6: Disk cache disabled. Please set disk cache directory in settings for best reliability\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from http://localhost\n"
- << "7: Response with service tickets for 2 destination(s) was successfully fetched from http://localhost\n"
- << "7: Response with service tickets for 1 destination(s) was successfully fetched from http://localhost\n"
- << "7: Got responses with service tickets with 3 pages for 5 destination(s)\n"
- << "3: Failed to get service ticket for dst=100500: Missing tvm_id in response, should never happend: 100500\n"
- << "3: Failed to get service ticket for dst=239: Dst is not found\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(FindMissingDsts) {
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector({6, 9}),
- TNotInitedUpdater::FindMissingDsts({1, 2, 3, 4}, {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector(),
- TNotInitedUpdater::FindMissingDsts({1, 2, 3, 4, 5, 6, 7, 8, 9}, {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector({1, 4, 6, 9}),
- TNotInitedUpdater::FindMissingDsts(NTvmApi::TDstSet(), {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector(1, 19),
- TNotInitedUpdater::FindMissingDsts({213}, {19, 213}));
-
- auto make = [](TVector<int> ids) {
- TServiceTickets::TMapIdStr m;
- for (auto i : ids) {
- m.insert({i, ""});
- }
- return MakeIntrusiveConst<TServiceTickets>(std::move(m), TServiceTickets::TMapIdStr{}, TServiceTickets::TMapAliasId{});
- };
-
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector({6, 9}),
- TNotInitedUpdater::FindMissingDsts(make({1, 2, 3, 4}), {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector(),
- TNotInitedUpdater::FindMissingDsts(make({1, 2, 3, 4, 5, 6, 7, 8, 9}), {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector({1, 4, 6, 9}),
- TNotInitedUpdater::FindMissingDsts(make({}), {1, 4, 6, 9}));
- UNIT_ASSERT_VALUES_EQUAL(NTvmApi::TClientSettings::TDstVector(1, 19),
- TNotInitedUpdater::FindMissingDsts(make({213}), {19, 213}));
- }
-
- Y_UNIT_TEST(CreateJsonArray) {
- UNIT_ASSERT_VALUES_EQUAL("[]", TNotInitedUpdater::CreateJsonArray({}));
- UNIT_ASSERT_VALUES_EQUAL("[sdlzkjvbsdljhfbsdajlhfbsakjdfb]",
- TNotInitedUpdater::CreateJsonArray({"sdlzkjvbsdljhfbsdajlhfbsakjdfb"}));
- UNIT_ASSERT_VALUES_EQUAL("[sdlzkjvbsdljhfbsdajlhfbsakjdfb,o92q83yh2uhq2eri23r]",
- TNotInitedUpdater::CreateJsonArray({"sdlzkjvbsdljhfbsdajlhfbsakjdfb",
- "o92q83yh2uhq2eri23r"}));
- }
-
- Y_UNIT_TEST(AppendArrayToJson) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(TNotInitedUpdater::AppendToJsonArray("", {}),
- yexception,
- "previous body required");
- UNIT_ASSERT_EXCEPTION_CONTAINS(TNotInitedUpdater::AppendToJsonArray("[kek", {}),
- yexception,
- "array is broken:");
-
- UNIT_ASSERT_VALUES_EQUAL("[kek]", TNotInitedUpdater::AppendToJsonArray("kek", {}));
-
- UNIT_ASSERT_VALUES_EQUAL(
- "[kek,sdlzkjvbsdljhfbsdajlhfbsakjdfb]",
- TNotInitedUpdater::AppendToJsonArray("kek",
- {"sdlzkjvbsdljhfbsdajlhfbsakjdfb"}));
- UNIT_ASSERT_VALUES_EQUAL(
- "[kek,sdlzkjvbsdljhfbsdajlhfbsakjdfb,o92q83yh2uhq2eri23r]",
- TNotInitedUpdater::AppendToJsonArray("kek",
- {"sdlzkjvbsdljhfbsdajlhfbsakjdfb", "o92q83yh2uhq2eri23r"}));
-
- UNIT_ASSERT_VALUES_EQUAL(
- "[kek,sdlzkjvbsdljhfbsdajlhfbsakjdfb]",
- TNotInitedUpdater::AppendToJsonArray("[kek]",
- {"sdlzkjvbsdljhfbsdajlhfbsakjdfb"}));
- UNIT_ASSERT_VALUES_EQUAL(
- "[kek,sdlzkjvbsdljhfbsdajlhfbsakjdfb,o92q83yh2uhq2eri23r]",
- TNotInitedUpdater::AppendToJsonArray("[kek]",
- {"sdlzkjvbsdljhfbsdajlhfbsakjdfb", "o92q83yh2uhq2eri23r"}));
- }
-
- Y_UNIT_TEST(UpdaterTimeouts) {
- NTvmApi::TClientSettings s;
- s.SetSelfTvmId(100500);
- s.EnableServiceTicketChecking();
- s.TvmHost = "localhost";
- s.TvmPort = GetRandomPort();
- const auto timeout = TDuration::MilliSeconds(10);
- s.TvmConnectTimeout = timeout;
- s.TvmSocketTimeout = timeout;
-
- {
- auto l = MakeIntrusive<TLogger>();
- auto startTs = ::Now();
- UNIT_ASSERT_EXCEPTION(NTvmApi::TThreadedUpdater::Create(s, l), yexception);
- UNIT_ASSERT_LT(::Now() - startTs, timeout * 2);
- }
- }
-}
-
-template <>
-void Out<TSmallVec<TString>>(IOutputStream& out, const TSmallVec<TString>& m) {
- for (const TString& s : m) {
- out << s << ";";
- }
-}
-
-template <>
-void Out<TServiceTickets::TMapIdStr>(
- IOutputStream& out,
- const TServiceTickets::TMapIdStr& m) {
- for (const auto& pair : m) {
- out << pair.first << " -> " << pair.second << ";";
- }
-}
-
-template <>
-void Out<NTestSuiteApiUpdater::TNotInitedUpdater::TPairTicketsErrors>(
- IOutputStream& out,
- const NTestSuiteApiUpdater::TNotInitedUpdater::TPairTicketsErrors& m) {
- out << m.Tickets << "\n";
- out << m.Errors << "\n";
-}
-
-template <>
-void Out<NTvmAuth::NTvmApi::TClientSettings::TDst>(IOutputStream& out, const NTvmAuth::NTvmApi::TClientSettings::TDst& m) {
- out << m.Id;
-}
diff --git a/library/cpp/tvmauth/client/ut/tvmtool_updater_ut.cpp b/library/cpp/tvmauth/client/ut/tvmtool_updater_ut.cpp
deleted file mode 100644
index 9435b46b38..0000000000
--- a/library/cpp/tvmauth/client/ut/tvmtool_updater_ut.cpp
+++ /dev/null
@@ -1,756 +0,0 @@
-#include "common.h"
-
-#include <library/cpp/tvmauth/client/facade.h>
-#include <library/cpp/tvmauth/client/misc/tool/threaded_updater.h>
-
-#include <library/cpp/http/simple/http_client.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/system/env.h>
-
-using namespace NTvmAuth;
-using namespace NTvmAuth::NTvmTool;
-
-Y_UNIT_TEST_SUITE(ToolUpdater) {
- static const TString SRV_TICKET = "3:serv:CBAQ__________9_IgYIexCUkQY:GioCM49Ob6_f80y6FY0XBVN4hLXuMlFeyMvIMiDuQnZkbkLpRpQOuQo5YjWoBjM0Vf-XqOm8B7xtrvxSYHDD7Q4OatN2l-Iwg7i71lE3scUeD36x47st3nd0OThvtjrFx_D8mw_c0GT5KcniZlqq1SjhLyAk1b_zJsx8viRAhCU";
- static const TString SRV_TICKET_DST_100503 = "3:serv:CBAQ__________9_IggIwMQHEJeRBg:Kj7VApP6D91UJ8pKpeaE3vYaNTBBJcdYpJLbF9w2-Mb-75s_SmMKkPqqA2rMS358uFfoYpv9YZxq0tIaUj5HPQ1WaQ1yiVuPZ_oi3pJRdr006eRyihM8PUfl6m9ioCFftfOcAg9oN5BGeHTNhn7VWuj3yMg7feaMB0zAUpyaPG0";
- static const TString TEST_TICKET = "3:user:CA0Q__________9_Gg4KAgh7EHsg0oXYzAQoAQ:FSADps3wNGm92Vyb1E9IVq5M6ZygdGdt1vafWWEhfDDeCLoVA-sJesxMl2pGW4OxJ8J1r_MfpG3ZoBk8rLVMHUFrPa6HheTbeXFAWl8quEniauXvKQe4VyrpA1SPgtRoFqi5upSDIJzEAe1YRJjq1EClQ_slMt8R0kA_JjKUX54";
- static const TString PROD_YATEAM_TICKET = "3:user:CAwQ__________9_Gg4KAgh7EHsg0oXYzAQoAg:G2wloFRSi8--RLb2GDSro_sKXPF2JSdL5CVOuOHgUcRvLm-3OxIPn0NUqbJ9DWDmhPplOqEiblIbLK85My1VMJ2aG5SLbRNKEtwfmxLvkwNpl_gUEwWPJm9_8Khslfj71P3hccxtEEqM9bJSMwHueVAY-a9HSzFo-uMFMeSgQ-k";
-
- class TMetaInfoProxy: public TMetaInfo {
- public:
- using TMetaInfo::ApplySettings;
- using TMetaInfo::BbEnvFromString;
- using TMetaInfo::Config_;
- using TMetaInfo::Fetch;
- using TMetaInfo::ParseMetaString;
- using TMetaInfo::TMetaInfo;
- };
-
- Y_UNIT_TEST(Settings) {
- NTvmTool::TClientSettings s("foo");
- UNIT_ASSERT_EXCEPTION_CONTAINS(s.SetAuthToken("\n "),
- TBrokenTvmClientSettings,
- "Auth token cannot be empty");
- UNIT_ASSERT_EXCEPTION_CONTAINS(s.GetAuthToken(),
- TBrokenTvmClientSettings,
- "Auth token cannot be empty. Env 'TVMTOOL_LOCAL_AUTHTOKEN' and 'QLOUD_TVM_TOKEN' are empty.");
-
- UNIT_ASSERT_NO_EXCEPTION(s.SetAuthToken(AUTH_TOKEN + "\n"));
- UNIT_ASSERT_VALUES_EQUAL(AUTH_TOKEN, s.GetAuthToken());
-
- UNIT_ASSERT_VALUES_EQUAL("localhost", s.GetHostname());
- UNIT_ASSERT_EXCEPTION_CONTAINS(s.SetHostname(""),
- TBrokenTvmClientSettings,
- "Hostname cannot be empty");
-
- UNIT_ASSERT_NO_EXCEPTION(s.SetHostname("qwe"));
- UNIT_ASSERT_VALUES_EQUAL("qwe", s.GetHostname());
- }
-
- Y_UNIT_TEST(SettingsCtor) {
- UNIT_ASSERT_EXCEPTION_CONTAINS(NTvmTool::TClientSettings(""),
- TBrokenTvmClientSettings,
- "Alias for your TVM client cannot be empty");
- {
- NTvmTool::TClientSettings s("self");
- UNIT_ASSERT_EXCEPTION_CONTAINS(s.GetAuthToken(),
- TBrokenTvmClientSettings,
- "Auth token cannot be empty. "
- "Env 'TVMTOOL_LOCAL_AUTHTOKEN' and 'QLOUD_TVM_TOKEN' are empty.");
- }
-
- struct TEnvs {
- TEnvs(const std::map<TString, TString>& Env) {
- for (const auto& [key, value] : Env) {
- Prev[key] = GetEnv(key);
- SetEnv(key, value);
- }
- }
-
- ~TEnvs() {
- for (const auto& [key, value] : Prev) {
- SetEnv(key, value);
- }
- }
-
- std::map<TString, TString> Prev;
- };
-
- struct TCase {
- std::map<TString, TString> Env;
- TString AuthToken;
- ui16 Port = 0;
- };
-
- std::vector<TCase> cases = {
- {
- {
- {"TVMTOOL_LOCAL_AUTHTOKEN", "qwerty"},
- },
- "qwerty",
- 1,
- },
- {
- {
- {"TVMTOOL_LOCAL_AUTHTOKEN", "qwerty"},
- {"QLOUD_TVM_TOKEN", "zxcvbn"},
- },
- "qwerty",
- 1,
- },
- {
- {
- {"QLOUD_TVM_TOKEN", "zxcvbn"},
- },
- "zxcvbn",
- 1,
- },
- {
- {
- {"TVMTOOL_LOCAL_AUTHTOKEN", "qwerty"},
- {"DEPLOY_TVM_TOOL_URL", "32272"},
- },
- "qwerty",
- 1,
- },
- {
- {
- {"TVMTOOL_LOCAL_AUTHTOKEN", "qwerty"},
- {"DEPLOY_TVM_TOOL_URL", "localhost:32272"},
- },
- "qwerty",
- 32272,
- },
- {
- {
- {"TVMTOOL_LOCAL_AUTHTOKEN", "qwerty"},
- {"DEPLOY_TVM_TOOL_URL", "http://localhost:32272"},
- },
- "qwerty",
- 32272,
- },
- };
-
- for (const TCase& c : cases) {
- TEnvs envs(c.Env);
-
- NTvmTool::TClientSettings s("self");
- UNIT_ASSERT_VALUES_EQUAL(c.AuthToken, s.GetAuthToken());
- UNIT_ASSERT_VALUES_EQUAL(c.Port, s.GetPort());
- }
- }
-
- Y_UNIT_TEST(Meta_Fetch) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
- TKeepAliveHttpClient client("localhost", port);
-
- TMetaInfoProxy m(nullptr);
- NTvmTool::TClientSettings settings("me");
- settings.SetAuthToken(AUTH_TOKEN);
- m.ApplySettings(settings);
-
- UNIT_ASSERT_VALUES_EQUAL(META, m.Fetch(client));
-
- settings.SetAuthToken("qwerty");
- m.ApplySettings(settings);
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.Fetch(client),
- TNonRetriableException,
- "Failed to fetch meta from tvmtool: localhost:");
-
- settings.SetAuthToken(AUTH_TOKEN);
- m.ApplySettings(settings);
- {
- TKeepAliveHttpClient client("localhost", 0);
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.Fetch(client),
- TRetriableException,
- "Failed to fetch meta data from tvmtool: ");
- }
-
- server.SetGenerator([]() {
- auto p = new TTvmTool;
- p->Code = HTTP_NOT_FOUND;
- return p; });
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.Fetch(client),
- TNonRetriableException,
- "Library does not support so old tvmtool. You need tvmtool>=1.1.0");
- server.SetGenerator([]() {
- auto p = new TTvmTool;
- p->Code = HTTP_INTERNAL_SERVER_ERROR;
- return p; });
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.Fetch(client),
- TRetriableException,
- "Failed to fetch meta from tvmtool: localhost:");
- }
-
- Y_UNIT_TEST(Meta_ParseMetaString_me) {
- TMetaInfo::TConfigPtr c;
- UNIT_ASSERT(c = TMetaInfoProxy::ParseMetaString(META, "me"));
- UNIT_ASSERT_VALUES_EQUAL(100500, c->SelfTvmId);
- UNIT_ASSERT_EQUAL(EBlackboxEnv::ProdYateam, c->BbEnv);
- UNIT_ASSERT_VALUES_EQUAL("", c->IdmSlug);
- UNIT_ASSERT_EQUAL(TMetaInfo::TDstAliases({{"bbox", 242}, {"pass_likers", 11}}), c->DstAliases);
- }
-
- Y_UNIT_TEST(Meta_ParseMetaString_tenant_with_roles) {
- TMetaInfo::TConfigPtr c;
- UNIT_ASSERT(c = TMetaInfoProxy::ParseMetaString(META, "tenant_with_roles"));
- UNIT_ASSERT_VALUES_EQUAL(100500, c->SelfTvmId);
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::ProdYateam, c->BbEnv);
- UNIT_ASSERT_VALUES_EQUAL("some_slug", c->IdmSlug);
- UNIT_ASSERT_VALUES_EQUAL(TMetaInfo::TDstAliases(), c->DstAliases);
- }
-
- Y_UNIT_TEST(Meta_ParseMetaString_pc) {
- TMetaInfo::TConfigPtr c;
- UNIT_ASSERT(c = TMetaInfoProxy::ParseMetaString(META, "push-client"));
- UNIT_ASSERT_VALUES_EQUAL(100501, c->SelfTvmId);
- UNIT_ASSERT_EQUAL(EBlackboxEnv::ProdYateam, c->BbEnv);
- UNIT_ASSERT_VALUES_EQUAL("", c->IdmSlug);
- UNIT_ASSERT_EQUAL(TMetaInfo::TDstAliases({{"pass_likers", 100502}}), c->DstAliases);
- }
-
- Y_UNIT_TEST(Meta_ParseMetaString_se) {
- TMetaInfo::TConfigPtr c;
- UNIT_ASSERT(c = TMetaInfoProxy::ParseMetaString(META, "something_else"));
- UNIT_ASSERT_VALUES_EQUAL(100503, c->SelfTvmId);
- UNIT_ASSERT_EQUAL(EBlackboxEnv::ProdYateam, c->BbEnv);
- UNIT_ASSERT_VALUES_EQUAL("", c->IdmSlug);
- UNIT_ASSERT(c->DstAliases.empty());
- }
-
- Y_UNIT_TEST(Meta_ParseMetaString_errors) {
- TMetaInfoProxy m(nullptr);
- UNIT_ASSERT(!m.ParseMetaString(META, "ololo"));
-
- TString meta = "}";
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.ParseMetaString(meta, "qqq"), yexception, meta);
- meta = "{}";
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.ParseMetaString(meta, "qqq"), yexception, meta);
- meta = R"({"tenants" : {}})";
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.ParseMetaString(meta, "qqq"), yexception, meta);
- meta = R"({"tenants" : [{"self":{}}]})";
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.ParseMetaString(meta, "qqq"), yexception, meta);
- }
-
- Y_UNIT_TEST(Meta_BbEnvFromString) {
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::Prod, TMetaInfoProxy::BbEnvFromString("Prod", META));
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::Test, TMetaInfoProxy::BbEnvFromString("Test", META));
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::ProdYateam, TMetaInfoProxy::BbEnvFromString("ProdYaTeam", META));
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::TestYateam, TMetaInfoProxy::BbEnvFromString("TestYaTeam", META));
- UNIT_ASSERT_VALUES_EQUAL(EBlackboxEnv::Stress, TMetaInfoProxy::BbEnvFromString("Stress", META));
- UNIT_ASSERT_EXCEPTION_CONTAINS(TMetaInfoProxy::BbEnvFromString("foo", META),
- yexception,
- "'bb_env'=='foo'");
- }
-
- Y_UNIT_TEST(Meta_ApplySettings) {
- NTvmTool::TClientSettings s("foo");
- s.SetAuthToken(AUTH_TOKEN);
-
- TMetaInfoProxy m(nullptr);
- m.ApplySettings(s);
-
- UNIT_ASSERT_VALUES_EQUAL(
- TKeepAliveHttpClient::THeaders({{"Authorization", AUTH_TOKEN}}),
- m.GetAuthHeader());
- }
-
- Y_UNIT_TEST(Meta_Init) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
- TKeepAliveHttpClient client("localhost", port);
-
- NTvmTool::TClientSettings s("me");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- auto l = MakeIntrusive<TLogger>();
- TMetaInfo m(l);
- UNIT_ASSERT_NO_EXCEPTION(m.Init(client, s));
- UNIT_ASSERT_VALUES_EQUAL(100500, m.GetConfig()->SelfTvmId);
- UNIT_ASSERT_EQUAL(EBlackboxEnv::ProdYateam, m.GetConfig()->BbEnv);
- UNIT_ASSERT_EQUAL(TMetaInfo::TDstAliases({{"bbox", 242}, {"pass_likers", 11}}), m.GetConfig()->DstAliases);
- UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]\n",
- l->Stream.Str());
- l->Stream.Clear();
- UNIT_ASSERT_VALUES_EQUAL(
- "/tvm/tickets?src=100500&dsts=11,242",
- TMetaInfo::GetRequestForTickets(*m.GetConfig()));
-
- server.SetGenerator([]() {
- auto p = new TTvmTool;
- p->Meta = R"({
- "bb_env" : "Prod",
- "tenants" : [{
- "self": {"alias" : "me", "client_id": 100500},
- "dsts" : [{"alias" : "pass_likers","client_id": 11}]
- }]
- })";
- return p; });
- UNIT_ASSERT(m.TryUpdateConfig(client));
- UNIT_ASSERT_VALUES_EQUAL(
- "6: Meta was updated. Old: (self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]). New: (self_tvm_id=100500, bb_env=Prod, idm_slug=<NULL>, dsts=[(pass_likers:11)])\n",
- l->Stream.Str());
- l->Stream.clear();
-
- s = NTvmTool::TClientSettings("foo");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- TMetaInfo m2(l);
- UNIT_ASSERT_EXCEPTION_CONTAINS(m2.Init(client, s), TNonRetriableException, "Alias 'foo' not found in meta info");
- UNIT_ASSERT_VALUES_EQUAL(TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n",
- l->Stream.Str());
- UNIT_ASSERT_EXCEPTION_CONTAINS(TMetaInfo::GetRequestForTickets({}),
- yexception,
- "DstAliases.empty()");
-
- server.SetGenerator([]() {
- auto p = new TTvmTool;
- p->Meta = "}";
- return p; });
- UNIT_ASSERT_EXCEPTION_CONTAINS(m.Init(client, s),
- TNonRetriableException,
- "Malformed json from tvmtool:");
- }
-
- class TNonInitedUpdater: public TThreadedUpdater {
- public:
- TNonInitedUpdater(const TString& host, ui16 port, TLoggerPtr logger)
- : TThreadedUpdater(host, port, TDuration::Seconds(5), TDuration::Seconds(30), logger)
- {
- }
-
- using TThreadedUpdater::ArePublicKeysOk;
- using TThreadedUpdater::AreServiceTicketsOk;
- using TThreadedUpdater::FetchPublicKeys;
- using TThreadedUpdater::FetchServiceTickets;
- using TThreadedUpdater::GetBirthTimeFromResponse;
- using TThreadedUpdater::Init;
- using TThreadedUpdater::IsTimeToUpdatePublicKeys;
- using TThreadedUpdater::IsTimeToUpdateServiceTickets;
- using TThreadedUpdater::LastVisitForConfig_;
- using TThreadedUpdater::MetaInfo_;
- using TThreadedUpdater::ParseFetchTicketsResponse;
- using TThreadedUpdater::SetBbEnv;
- using TThreadedUpdater::SetServiceContext;
- using TThreadedUpdater::SetServiceTickets;
- using TThreadedUpdater::SetUpdateTimeOfPublicKeys;
- using TThreadedUpdater::SetUpdateTimeOfServiceTickets;
- using TThreadedUpdater::SetUserContext;
- using TThreadedUpdater::TPairTicketsErrors;
- using TThreadedUpdater::UpdateKeys;
- using TThreadedUpdater::UpdateServiceTickets;
- };
-
- Y_UNIT_TEST(GetBirthTimeFromResponse) {
- THttpHeaders h;
- UNIT_ASSERT_EXCEPTION_CONTAINS(TNonInitedUpdater::GetBirthTimeFromResponse(h, "ololo"),
- yexception,
- "Failed to fetch bithtime of ololo from tvmtool");
-
- h.AddHeader(THttpInputHeader("X-Ya-Tvmtool-Data-Birthtime: qwe"));
- UNIT_ASSERT_EXCEPTION_CONTAINS(TNonInitedUpdater::GetBirthTimeFromResponse(h, "ololo"),
- yexception,
- "Bithtime of ololo from tvmtool must be unixtime. Got: qwe");
-
- h.AddOrReplaceHeader(THttpInputHeader("X-Ya-Tvmtool-Data-Birthtime: 123"));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(123), TNonInitedUpdater::GetBirthTimeFromResponse(h, "ololo"));
- }
-
- Y_UNIT_TEST(Fetch) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
- TKeepAliveHttpClient client("localhost", port);
-
- auto l = MakeIntrusive<TLogger>();
- TNonInitedUpdater u("localhost", port, l);
- NTvmTool::TClientSettings s("me");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- u.MetaInfo_.Init(client, s);
- auto p = u.FetchPublicKeys();
- UNIT_ASSERT_STRINGS_EQUAL(NUnittest::TVMKNIFE_PUBLIC_KEYS, p.first);
- UNIT_ASSERT_VALUES_EQUAL(BIRTHTIME, p.second);
-
- auto p2 = u.FetchServiceTickets(*u.MetaInfo_.GetConfig());
- UNIT_ASSERT_STRINGS_EQUAL(TICKETS_ME, p2.first);
- UNIT_ASSERT_VALUES_EQUAL(BIRTHTIME, p2.second);
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(ParseFetchTicketsResponse) {
- auto l = MakeIntrusive<TLogger>();
- TNonInitedUpdater u("", 0, l);
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(u.ParseFetchTicketsResponse("}", {}),
- yexception,
- "Invalid json from tvmtool: }");
-
- auto t = u.ParseFetchTicketsResponse(TICKETS_ME, {{"pass_likers", 11}, {"se", 2}});
- auto expected = TNonInitedUpdater::TPairTicketsErrors{
- {{11, "3:serv:CBAQ__________9_IgYIlJEGEAs:T-apeMNWFc_vHPQ3iLaZv9NjG-hf5-i23O4AhRu1M68ryN3FU5qvyqTSSiPbtJdFP6EE41QQBzEs59dHn9DRkqQNwwKf1is00Oewwj2XKO0uHukuzd9XxZnro7MfjPswsjWufxX28rmJtlfSXwAtyKt8TI5yKJnMeBPQ0m5R3k8"}},
- {
- {2, "Missing tvm_id in response, should never happend: se"},
- },
- };
- UNIT_ASSERT_VALUES_EQUAL(expected, t);
-
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "3: Failed to get ServiceTicket for se (2): Missing tvm_id in response, should never happend: se\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(Update) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
- TKeepAliveHttpClient client("localhost", port);
-
- auto l = MakeIntrusive<TLogger>();
- TNonInitedUpdater u("localhost", port, l);
- NTvmTool::TClientSettings s("me");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- u.MetaInfo_.Init(client, s);
-
- using TTickets = TServiceTickets::TMapAliasStr;
- UNIT_ASSERT(!u.GetCachedServiceTickets());
- UNIT_ASSERT_VALUES_EQUAL(BIRTHTIME, u.UpdateServiceTickets(*u.MetaInfo_.GetConfig()));
- UNIT_ASSERT(u.GetCachedServiceTickets());
- UNIT_ASSERT_VALUES_EQUAL(TInstant(), u.GetUpdateTimeOfServiceTickets());
- UNIT_ASSERT_EQUAL(
- TTickets({
- {"bbox", "3:serv:CBAQ__________9_IgcIlJEGEPIB:N7luw0_rVmBosTTI130jwDbQd0-cMmqJeEl0ma4ZlIo_mHXjBzpOuMQ3A9YagbmOBOt8TZ_gzGvVSegWZkEeB24gM22acw0w-RcHaQKrzSOA5Zq8WLNIC8QUa4_WGTlAsb7R7eC4KTAGgouIquNAgMBdTuGOuZHnMLvZyLnOMKc"},
- {"pass_likers", "3:serv:CBAQ__________9_IgYIlJEGEAs:T-apeMNWFc_vHPQ3iLaZv9NjG-hf5-i23O4AhRu1M68ryN3FU5qvyqTSSiPbtJdFP6EE41QQBzEs59dHn9DRkqQNwwKf1is00Oewwj2XKO0uHukuzd9XxZnro7MfjPswsjWufxX28rmJtlfSXwAtyKt8TI5yKJnMeBPQ0m5R3k8"},
- }),
- u.GetCachedServiceTickets()->TicketsByAlias);
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]\n",
- l->Stream.Str());
- l->Stream.Clear();
-
- UNIT_ASSERT(!u.GetCachedServiceContext());
- UNIT_ASSERT(!u.GetCachedUserContext());
- UNIT_ASSERT_VALUES_EQUAL(BIRTHTIME, u.UpdateKeys(*u.MetaInfo_.GetConfig()));
- UNIT_ASSERT(u.GetCachedServiceContext());
- UNIT_ASSERT(!u.GetCachedUserContext());
- u.SetBbEnv(EBlackboxEnv::Test);
- UNIT_ASSERT(u.GetCachedUserContext());
- UNIT_ASSERT_VALUES_EQUAL("", l->Stream.Str());
- l->Stream.Clear();
-
- {
- TAsyncUpdaterPtr u = TThreadedUpdater::Create(s, l);
- UNIT_ASSERT(u->GetCachedServiceTickets());
- UNIT_ASSERT(u->GetCachedServiceContext());
- UNIT_ASSERT(u->GetCachedUserContext());
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus());
-
- NTvmAuth::TTvmClient c(u);
- UNIT_ASSERT(c.CheckServiceTicket(SRV_TICKET));
- UNIT_ASSERT(!c.CheckServiceTicket(SRV_TICKET_DST_100503));
- UNIT_ASSERT(c.CheckUserTicket(PROD_YATEAM_TICKET));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:CBAQ__________9_IgYIlJEGEAs:T-apeMNWFc_vHPQ3iLaZv9NjG-hf5-i23O4AhRu1M68ryN3FU5qvyqTSSiPbtJdFP6EE41QQBzEs59dHn9DRkqQNwwKf1is00Oewwj2XKO0uHukuzd9XxZnro7MfjPswsjWufxX28rmJtlfSXwAtyKt8TI5yKJnMeBPQ0m5R3k8", c.GetServiceTicketFor("pass_likers"));
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]\n"
- << "7: Tickets fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Public keys fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- l->Stream.Clear();
-
- {
- NTvmTool::TClientSettings s("something_else");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
-
- TAsyncUpdaterPtr u = TThreadedUpdater::Create(s, l);
- UNIT_ASSERT(!u->GetCachedServiceTickets());
- UNIT_ASSERT(u->GetCachedServiceContext());
- UNIT_ASSERT(u->GetCachedUserContext());
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus());
-
- NTvmAuth::TTvmClient c(u);
- UNIT_ASSERT(!c.CheckServiceTicket(SRV_TICKET));
- UNIT_ASSERT(c.CheckServiceTicket(SRV_TICKET_DST_100503));
- UNIT_ASSERT(c.CheckUserTicket(PROD_YATEAM_TICKET));
- UNIT_ASSERT_EXCEPTION_CONTAINS(c.GetServiceTicketFor("pass_likers"),
- TBrokenTvmClientSettings,
- "Need to enable ServiceTickets fetching");
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100503, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[]\n"
- << "7: Public keys fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- l->Stream.Clear();
- }
-
- Y_UNIT_TEST(IsOk) {
- TNonInitedUpdater u("", 0, TDevNullLogger::IAmBrave());
- using TTickets = TServiceTickets::TMapIdStr;
-
- UNIT_ASSERT(u.AreServiceTicketsOk(0));
- UNIT_ASSERT(!u.AreServiceTicketsOk(2));
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(TTickets(),
- TTickets(),
- TServiceTickets::TMapAliasId()));
- UNIT_ASSERT(u.AreServiceTicketsOk(0));
- UNIT_ASSERT(!u.AreServiceTicketsOk(2));
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- TTickets({
- {1, "mega_ticket"},
- {2, "mega_ticket2"},
- }),
- TTickets({
- {3, "mega_error3"},
- }),
- TServiceTickets::TMapAliasId()));
- UNIT_ASSERT(u.AreServiceTicketsOk(0));
- UNIT_ASSERT(!u.AreServiceTicketsOk(2));
-
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- TTickets({
- {1, "mega_ticket"},
- {2, "mega_ticket2"},
- }),
- TTickets({
- {3, "mega_error3"},
- }),
- TServiceTickets::TMapAliasId({
- {"mega_ticket", 1},
- {"mega_ticket2", 2},
- {"mega_ticket3", 3},
- })));
- UNIT_ASSERT(u.AreServiceTicketsOk(2));
-
- UNIT_ASSERT(!u.ArePublicKeysOk());
- u.SetServiceContext(MakeIntrusiveConst<TServiceContext>(
- TServiceContext::CheckingFactory(12, NUnittest::TVMKNIFE_PUBLIC_KEYS)));
- UNIT_ASSERT(!u.ArePublicKeysOk());
- u.SetUserContext(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT(!u.ArePublicKeysOk());
- u.SetBbEnv(EBlackboxEnv::Test);
- UNIT_ASSERT(u.ArePublicKeysOk());
- }
-
- Y_UNIT_TEST(IsTimeToUpdate) {
- TNonInitedUpdater u("", 0, TDevNullLogger::IAmBrave());
-
- UNIT_ASSERT(!u.IsTimeToUpdatePublicKeys(TInstant::Now() - TDuration::Seconds(597)));
- UNIT_ASSERT(u.IsTimeToUpdatePublicKeys(TInstant::Now() - TDuration::Seconds(603)));
-
- TMetaInfo::TConfig cfg;
- UNIT_ASSERT(!u.IsTimeToUpdateServiceTickets(cfg, TInstant::Now() - TDuration::Seconds(597)));
- UNIT_ASSERT(!u.IsTimeToUpdateServiceTickets(cfg, TInstant::Now() - TDuration::Seconds(603)));
-
- cfg.DstAliases = {{"q", 1}};
- UNIT_ASSERT(!u.IsTimeToUpdateServiceTickets(cfg, TInstant::Now() - TDuration::Seconds(597)));
- UNIT_ASSERT(u.IsTimeToUpdateServiceTickets(cfg, TInstant::Now() - TDuration::Seconds(603)));
- }
-
- Y_UNIT_TEST(InitWithOldData) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port,
- []() {
- auto p = new TTvmTool;
- p->Birthtime = TInstant::Seconds(123);
- return p;
- });
-
- NTvmTool::TClientSettings s("me");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
-
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TThreadedUpdater::Create(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry: ");
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100500, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:11)(bbox:242)]\n"
- << "4: Error while fetching of tickets: Service tickets are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Service tickets have not been refreshed for too long period\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "4: Error while fetching of tickets: Service tickets are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Service tickets have not been refreshed for too long period\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "4: Error while fetching of tickets: Service tickets are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Service tickets have not been refreshed for too long period\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(InitWithOldData_onlyKeys) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port,
- []() {
- auto p = new TTvmTool;
- p->Birthtime = TInstant::Seconds(123);
- return p;
- });
-
- NTvmTool::TClientSettings s("something_else");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
-
- {
- s.OverrideBlackboxEnv(EBlackboxEnv::Stress);
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TThreadedUpdater::Create(s, l),
- TBrokenTvmClientSettings,
- "Overriding of BlackboxEnv is illegal: ProdYateam -> Stress");
- }
-
- s.OverrideBlackboxEnv(EBlackboxEnv::Prod);
- auto l = MakeIntrusive<TLogger>();
- UNIT_ASSERT_EXCEPTION_CONTAINS(TThreadedUpdater::Create(s, l),
- TRetriableException,
- "Failed to start TvmClient. You can retry: ");
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100503, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[]\n"
- << "6: Meta: override blackbox env: ProdYateam->Prod\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n"
- << "4: Error while fetching of public keys: Public keys are too old: 1970-01-01T00:02:03.000000Z\n"
- << "3: Public keys have not been refreshed for too long period\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(Init) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
-
- NTvmTool::TClientSettings s("push-client");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- s.SetHostname("localhost");
-
- auto l = MakeIntrusive<TLogger>();
- {
- TAsyncUpdaterPtr u = TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100501, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_likers:100502)]\n"
- << "7: Tickets fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Public keys fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- }
-
- Y_UNIT_TEST(InitWithoutTvmtool) {
- NTvmTool::TClientSettings s("me");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(0);
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(TThreadedUpdater::Create(s, TDevNullLogger::IAmBrave()),
- TNonRetriableException,
- "can not connect to ");
- }
-
- Y_UNIT_TEST(GetStatus) {
- TNonInitedUpdater u("", 0, TDevNullLogger::IAmBrave());
- TMetaInfoProxy m(nullptr);
- m.Config_ = std::make_shared<TMetaInfo::TConfig>();
- u.MetaInfo_ = m;
- u.LastVisitForConfig_ = TInstant::Now();
-
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Error, u.GetStatus());
- u.SetUpdateTimeOfPublicKeys(TInstant::Now() - TDuration::Days(3));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, u.GetStatus());
- u.SetUpdateTimeOfPublicKeys(TInstant::Now() - TDuration::Hours(3));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
-
- u.SetServiceTickets(new TServiceTickets({}, {}, {}));
-
- TMetaInfo::TConfig cfg;
- cfg.DstAliases = {{"q", 1}, {"q2", 2}};
- m.Config_ = std::make_shared<TMetaInfo::TConfig>(cfg);
- u.MetaInfo_ = m;
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Error, u.GetStatus());
- u.SetUpdateTimeOfServiceTickets(TInstant::Now() - TDuration::Hours(3));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Error, u.GetStatus());
-
- u.SetServiceTickets(MakeIntrusiveConst<TServiceTickets>(
- TServiceTickets::TMapIdStr({{1, "3:serv:CBAQ__________9_IgYIKhCUkQY:CX"}, {2, "t"}}),
- TServiceTickets::TMapIdStr({{3, "mega_error"}, {4, "error2"}}),
- TServiceTickets::TMapAliasId({
- {"some_alias#1", 1},
- {"some_alias#2", 2},
- })));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, u.GetStatus());
-
- const TInstant* inv = &u.GetCachedServiceTickets()->InvalidationTime;
- *const_cast<TInstant*>(inv) = TInstant::Now() + TDuration::Hours(3);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, u.GetStatus());
-
- u.SetUpdateTimeOfServiceTickets(TInstant::Now() - TDuration::Minutes(3));
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u.GetStatus());
-
- u.LastVisitForConfig_ = TInstant::Now() - TDuration::Minutes(1);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Warning, u.GetStatus());
- }
-
- Y_UNIT_TEST(multiNamesForDst) {
- TPortManager pm;
- ui16 port = pm.GetPort(80);
- NMock::TMockServer server(port, []() { return new TTvmTool; });
-
- NTvmTool::TClientSettings s("multi_names_for_dst");
- s.SetAuthToken(AUTH_TOKEN);
- s.SetPort(port);
- s.SetHostname("localhost");
-
- auto l = MakeIntrusive<TLogger>();
- {
- TAsyncUpdaterPtr u = TThreadedUpdater::Create(s, l);
- UNIT_ASSERT_VALUES_EQUAL(TClientStatus::Ok, u->GetStatus());
- }
- UNIT_ASSERT_VALUES_EQUAL(
- TStringBuilder()
- << "7: Meta info fetched from localhost:" << port << "\n"
- << "6: Meta: self_tvm_id=100599, bb_env=ProdYateam, idm_slug=<NULL>, dsts=[(pass_haters:100502)(pass_likers:100502)]\n"
- << "7: Tickets fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Public keys fetched from tvmtool: 2425-09-17T11:04:00.000000Z\n"
- << "7: Thread-worker started\n"
- << "7: Thread-worker stopped\n",
- l->Stream.Str());
- }
-}
diff --git a/library/cpp/tvmauth/client/ut/utils_ut.cpp b/library/cpp/tvmauth/client/ut/utils_ut.cpp
deleted file mode 100644
index e780fb2779..0000000000
--- a/library/cpp/tvmauth/client/ut/utils_ut.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-#include <library/cpp/tvmauth/client/misc/utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-Y_UNIT_TEST_SUITE(UtilsTest) {
- using namespace NTvmAuth;
-
- Y_UNIT_TEST(ParseDstMap) {
- using TMap = NTvmAuth::NTvmApi::TClientSettings::TDstMap;
- UNIT_ASSERT_EQUAL(TMap(), NUtils::ParseDstMap(""));
- UNIT_ASSERT_EXCEPTION(NUtils::ParseDstMap(";"), TFromStringException);
- UNIT_ASSERT_EXCEPTION(NUtils::ParseDstMap(":"), TFromStringException);
- UNIT_ASSERT_EXCEPTION(NUtils::ParseDstMap("3;"), TFromStringException);
- UNIT_ASSERT_EXCEPTION(NUtils::ParseDstMap("3:foo;"), TFromStringException);
-
- UNIT_ASSERT_EQUAL(TMap({
- {"foo", 3},
- }),
- NUtils::ParseDstMap("foo:3"));
- UNIT_ASSERT_EQUAL(TMap({
- {"foo", 3},
- {"bar", 17},
- }),
- NUtils::ParseDstMap("foo:3;bar:17;"));
- }
-
- Y_UNIT_TEST(ParseDstVector) {
- using TVector = NTvmAuth::NTvmApi::TClientSettings::TDstVector;
- UNIT_ASSERT_EQUAL(TVector(), NUtils::ParseDstVector(""));
- UNIT_ASSERT_EXCEPTION_CONTAINS(NUtils::ParseDstVector(";"),
- yexception,
- "Cannot parse empty string as number");
- UNIT_ASSERT_EXCEPTION_CONTAINS(NUtils::ParseDstVector(":"),
- yexception,
- "Unexpected symbol");
- UNIT_ASSERT_EXCEPTION_CONTAINS(NUtils::ParseDstVector("3:foo;"),
- yexception,
- "Unexpected symbol");
- UNIT_ASSERT_EXCEPTION_CONTAINS(NUtils::ParseDstVector("foo:3;"),
- yexception,
- "Unexpected symbol");
-
- UNIT_ASSERT_EQUAL(TVector(1, 3),
- NUtils::ParseDstVector("3"));
- UNIT_ASSERT_EQUAL(TVector({3, 17}),
- NUtils::ParseDstVector("3;17;"));
- }
-
- Y_UNIT_TEST(ToHex) {
- UNIT_ASSERT_VALUES_EQUAL("", NUtils::ToHex(""));
- UNIT_ASSERT_VALUES_EQUAL("61", NUtils::ToHex("a"));
- UNIT_ASSERT_VALUES_EQUAL(
- "6C6B787A6E7620736C6A6876627761656220",
- NUtils::ToHex("lkxznv sljhvbwaeb "));
- }
-
- Y_UNIT_TEST(CheckBbEnvOverriding) {
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Prod, EBlackboxEnv::Prod));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Prod, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Prod, EBlackboxEnv::Test));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Prod, EBlackboxEnv::TestYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Prod, EBlackboxEnv::Stress));
-
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::ProdYateam, EBlackboxEnv::Prod));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::ProdYateam, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::ProdYateam, EBlackboxEnv::Test));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::ProdYateam, EBlackboxEnv::TestYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::ProdYateam, EBlackboxEnv::Stress));
-
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Test, EBlackboxEnv::Prod));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Test, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Test, EBlackboxEnv::Test));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Test, EBlackboxEnv::TestYateam));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Test, EBlackboxEnv::Stress));
-
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::TestYateam, EBlackboxEnv::Prod));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::TestYateam, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::TestYateam, EBlackboxEnv::Test));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::TestYateam, EBlackboxEnv::TestYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::TestYateam, EBlackboxEnv::Stress));
-
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Stress, EBlackboxEnv::Prod));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Stress, EBlackboxEnv::ProdYateam));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Stress, EBlackboxEnv::Test));
- UNIT_ASSERT(!NUtils::CheckBbEnvOverriding(EBlackboxEnv::Stress, EBlackboxEnv::TestYateam));
- UNIT_ASSERT(NUtils::CheckBbEnvOverriding(EBlackboxEnv::Stress, EBlackboxEnv::Stress));
- }
-}
diff --git a/library/cpp/tvmauth/deprecated/README.md b/library/cpp/tvmauth/deprecated/README.md
deleted file mode 100644
index d9ea09c3c0..0000000000
--- a/library/cpp/tvmauth/deprecated/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Please don't use this part of library directly.
-Please use [TTvmClient](https://a.yandex-team.ru/arc/trunk/arcadia/library/cpp/tvmauth/README.md) instead.
diff --git a/library/cpp/tvmauth/deprecated/service_context.cpp b/library/cpp/tvmauth/deprecated/service_context.cpp
deleted file mode 100644
index 24822a9d53..0000000000
--- a/library/cpp/tvmauth/deprecated/service_context.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/src/service_impl.h>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "ServiceContext already moved out";
-
- TServiceContext::TServiceContext(TStringBuf secretBase64, TTvmId selfTvmId, TStringBuf tvmKeysResponse)
- : Impl_(MakeHolder<TImpl>(secretBase64, selfTvmId, tvmKeysResponse))
- {
- }
-
- TServiceContext::TServiceContext(TServiceContext&& o) = default;
- TServiceContext& TServiceContext::operator=(TServiceContext&& o) = default;
- TServiceContext::~TServiceContext() = default;
-
- TServiceContext TServiceContext::CheckingFactory(TTvmId selfTvmId, TStringBuf tvmKeysResponse) {
- TServiceContext c;
- c.Impl_ = MakeHolder<TImpl>(selfTvmId, tvmKeysResponse);
- return c;
- }
-
- TServiceContext TServiceContext::SigningFactory(TStringBuf secretBase64) {
- TServiceContext c;
- c.Impl_ = MakeHolder<TImpl>(secretBase64);
- return c;
- }
-
- TCheckedServiceTicket TServiceContext::Check(TStringBuf ticketBody) const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->Check(ticketBody);
- }
-
- TString TServiceContext::SignCgiParamsForTvm(TStringBuf ts, TStringBuf dst, TStringBuf scopes) const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->SignCgiParamsForTvm(ts, dst, scopes);
- }
-}
diff --git a/library/cpp/tvmauth/deprecated/service_context.h b/library/cpp/tvmauth/deprecated/service_context.h
deleted file mode 100644
index bc14d381b2..0000000000
--- a/library/cpp/tvmauth/deprecated/service_context.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-
-#include <util/generic/ptr.h>
-
-namespace NTvmAuth {
- class TServiceContext: public TAtomicRefCount<TServiceContext> {
- public:
- /*!
- * Create service context. Serivce contexts are used to store TVM keys and parse service tickets.
- * @param selfTvmId
- * @param secretBase64
- * @param tvmKeysResponse
- */
- TServiceContext(TStringBuf secretBase64, TTvmId selfTvmId, TStringBuf tvmKeysResponse);
- TServiceContext(TServiceContext&&);
- ~TServiceContext();
-
- /*!
- * Create service context only for checking service tickets
- * \param[in] selfTvmId
- * \param[in] tvmKeysResponse
- * \return
- */
- static TServiceContext CheckingFactory(TTvmId selfTvmId, TStringBuf tvmKeysResponse);
-
- /*!
- * Create service context only for signing HTTP request to TVM-API
- * \param[in] secretBase64
- * \return
- */
- static TServiceContext SigningFactory(TStringBuf secretBase64);
-
- TServiceContext& operator=(TServiceContext&&);
-
- /*!
- * Parse and validate service ticket body then create TCheckedServiceTicket object.
- * @param ticketBody
- * @return TCheckedServiceTicket object
- */
- TCheckedServiceTicket Check(TStringBuf ticketBody) const;
-
- /*!
- * Sign params for TVM API
- * @param ts Param 'ts' of request to TVM
- * @param dst Param 'dst' of request to TVM
- * @param scopes Param 'scopes' of request to TVM
- * @return Signed string
- */
- TString SignCgiParamsForTvm(TStringBuf ts, TStringBuf dst, TStringBuf scopes = TStringBuf()) const;
-
- class TImpl;
-
- private:
- TServiceContext() = default;
-
- private:
- THolder<TImpl> Impl_;
- };
-
- using TServiceContextPtr = TIntrusiveConstPtr<TServiceContext>;
-}
diff --git a/library/cpp/tvmauth/deprecated/user_context.cpp b/library/cpp/tvmauth/deprecated/user_context.cpp
deleted file mode 100644
index 712f622f1a..0000000000
--- a/library/cpp/tvmauth/deprecated/user_context.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-#include <library/cpp/tvmauth/src/user_impl.h>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "UserContext already moved out";
-
- TUserContext::TUserContext(EBlackboxEnv env, TStringBuf tvmKeysResponse)
- : Impl_(MakeHolder<TImpl>(env, tvmKeysResponse))
- {
- }
-
- TUserContext::TUserContext(TUserContext&& o) = default;
- TUserContext& TUserContext::operator=(TUserContext&& o) = default;
- TUserContext::~TUserContext() = default;
-
- TCheckedUserTicket TUserContext::Check(TStringBuf ticketBody) const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->Check(ticketBody);
- }
-}
diff --git a/library/cpp/tvmauth/deprecated/user_context.h b/library/cpp/tvmauth/deprecated/user_context.h
deleted file mode 100644
index f7fe67d02e..0000000000
--- a/library/cpp/tvmauth/deprecated/user_context.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-#include <util/generic/ptr.h>
-
-namespace NTvmAuth {
- class TUserContext: public TAtomicRefCount<TUserContext> {
- public:
- TUserContext(EBlackboxEnv env, TStringBuf tvmKeysResponse);
- TUserContext(TUserContext&&);
- ~TUserContext();
-
- TUserContext& operator=(TUserContext&&);
-
- /*!
- * Parse and validate user ticket body then create TCheckedUserTicket object.
- * @param ticketBody
- * @return TCheckedUserTicket object
- */
- TCheckedUserTicket Check(TStringBuf ticketBody) const;
-
- class TImpl;
-
- private:
- THolder<TImpl> Impl_;
- };
-
- using TUserContextPtr = TIntrusiveConstPtr<TUserContext>;
-}
diff --git a/library/cpp/tvmauth/exception.h b/library/cpp/tvmauth/exception.h
deleted file mode 100644
index f528886b95..0000000000
--- a/library/cpp/tvmauth/exception.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include <util/generic/yexception.h>
-
-#include <exception>
-
-namespace NTvmAuth {
- class TTvmException: public yexception {
- };
- class TContextException: public TTvmException {
- };
- class TMalformedTvmSecretException: public TContextException {
- };
- class TMalformedTvmKeysException: public TContextException {
- };
- class TEmptyTvmKeysException: public TContextException {
- };
- class TNotAllowedException: public TTvmException {
- };
-}
diff --git a/library/cpp/tvmauth/src/parser.cpp b/library/cpp/tvmauth/src/parser.cpp
deleted file mode 100644
index 358de58d36..0000000000
--- a/library/cpp/tvmauth/src/parser.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-#include "parser.h"
-
-#include "utils.h"
-
-#include <library/cpp/tvmauth/exception.h>
-
-#include <util/generic/strbuf.h>
-#include <util/string/split.h>
-
-#include <ctime>
-
-namespace NTvmAuth {
- TString TParserTvmKeys::ParseStrV1(TStringBuf str) {
- while (str && str.back() == '\n') {
- str.Chop(1);
- }
-
- TStringBuf ver = str.NextTok(DELIM);
- if (!str || !ver || ver != "1") {
- throw TMalformedTvmKeysException() << "Malformed TVM keys";
- }
- TString res = NUtils::Base64url2bin(str);
- if (res.empty()) {
- throw TMalformedTvmKeysException() << "Malformed TVM keys";
- }
- return res;
- }
-
- TStringBuf TParserTickets::UserFlag() {
- static const char BUF_[] = "user";
- return TStringBuf(BUF_, sizeof(BUF_) - 1);
- }
-
- TStringBuf TParserTickets::ServiceFlag() {
- static const char BUF_[] = "serv";
- return TStringBuf(BUF_, sizeof(BUF_) - 1);
- }
-
- TParserTickets::TRes TParserTickets::ParseV3(TStringBuf body, const NRw::TPublicKeys& keys, TStringBuf type) {
- TStrRes str = ParseStrV3(body, type);
- TRes res(str.Status);
- if (str.Status != ETicketStatus::Ok) {
- return TRes(str.Status);
- }
- if (!res.Ticket.ParseFromString(str.Proto)) {
- res.Status = ETicketStatus::Malformed;
- return res;
- }
- if (res.Ticket.expirationtime() <= time(nullptr)) {
- res.Status = ETicketStatus::Expired;
- return res;
- }
-
- auto itKey = keys.find(res.Ticket.keyid());
- if (itKey == keys.end()) {
- res.Status = ETicketStatus::MissingKey;
- return res;
- }
- if (!itKey->second.CheckSign(str.ForCheck, str.Sign)) {
- res.Status = ETicketStatus::SignBroken;
- return res;
- }
- return res;
- }
-
- TParserTickets::TStrRes TParserTickets::ParseStrV3(TStringBuf body, TStringBuf type) {
- TStringBuf forCheck = body;
- TStringBuf version = body.NextTok(DELIM);
- if (!body || version.size() != 1) {
- return {ETicketStatus::Malformed, {}, {}, {}};
- }
- if (version != "3") {
- return {ETicketStatus::UnsupportedVersion, {}, {}, {}};
- }
-
- TStringBuf ticketType = body.NextTok(DELIM);
- if (ticketType != type) {
- return {ETicketStatus::InvalidTicketType, {}, {}, {}};
- }
-
- TStringBuf proto = body.NextTok(DELIM);
- TStringBuf sign = body.NextTok(DELIM);
-
- if (!proto || !sign || body.size() > 0) {
- return {ETicketStatus::Malformed, {}, {}, {}};
- }
-
- TString protoBin = NUtils::Base64url2bin(proto);
- TString signBin = NUtils::Base64url2bin(sign);
-
- if (!protoBin || !signBin) {
- return {ETicketStatus::Malformed, {}, {}, {}};
- }
-
- return {ETicketStatus::Ok, std::move(protoBin), std::move(signBin), forCheck.Chop(sign.size())};
- }
-}
diff --git a/library/cpp/tvmauth/src/parser.h b/library/cpp/tvmauth/src/parser.h
deleted file mode 100644
index 678e709444..0000000000
--- a/library/cpp/tvmauth/src/parser.h
+++ /dev/null
@@ -1,51 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/src/protos/ticket2.pb.h>
-#include <library/cpp/tvmauth/src/rw/keys.h>
-
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <util/generic/fwd.h>
-
-#include <string>
-
-namespace NTvmAuth {
- struct TParserTvmKeys {
- static inline const char DELIM = ':';
- static TString ParseStrV1(TStringBuf str);
- };
-
- struct TParserTickets {
- static const char DELIM = ':';
-
- static TStringBuf UserFlag();
- static TStringBuf ServiceFlag();
-
- struct TRes {
- TRes(ETicketStatus status)
- : Status(status)
- {
- }
-
- ETicketStatus Status;
-
- ticket2::Ticket Ticket;
- };
- static TRes ParseV3(TStringBuf body, const NRw::TPublicKeys& keys, TStringBuf type);
-
- // private:
- struct TStrRes {
- const ETicketStatus Status;
-
- TString Proto;
- TString Sign;
-
- TStringBuf ForCheck;
-
- bool operator==(const TStrRes& o) const { // for tests
- return Status == o.Status && Proto == o.Proto && Sign == o.Sign && ForCheck == o.ForCheck;
- }
- };
- static TStrRes ParseStrV3(TStringBuf body, TStringBuf type);
- };
-}
diff --git a/library/cpp/tvmauth/src/protos/CMakeLists.txt b/library/cpp/tvmauth/src/protos/CMakeLists.txt
deleted file mode 100644
index 128debac24..0000000000
--- a/library/cpp/tvmauth/src/protos/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-
-add_library(tvmauth-src-protos)
-target_link_libraries(tvmauth-src-protos PUBLIC
- contrib-libs-cxxsupp
- yutil
- contrib-libs-protobuf
-)
-target_proto_messages(tvmauth-src-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/protos/ticket2.proto
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/protos/tvm_keys.proto
-)
-target_proto_addincls(tvmauth-src-protos
- ./
- ${CMAKE_SOURCE_DIR}/
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
- ${CMAKE_BINARY_DIR}
- ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
-)
-target_proto_outs(tvmauth-src-protos
- --cpp_out=${CMAKE_BINARY_DIR}/
- --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
-)
diff --git a/library/cpp/tvmauth/src/protos/ticket2.proto b/library/cpp/tvmauth/src/protos/ticket2.proto
deleted file mode 100644
index 66c00a7d01..0000000000
--- a/library/cpp/tvmauth/src/protos/ticket2.proto
+++ /dev/null
@@ -1,31 +0,0 @@
-package ticket2;
-
-option go_package = "a.yandex-team.ru/library/cpp/tvmauth/src/protos";
-
-import "library/cpp/tvmauth/src/protos/tvm_keys.proto";
-
-message User {
- required uint64 uid = 1;
-}
-
-message UserTicket {
- repeated User users = 1;
- required uint64 defaultUid = 2;
- repeated string scopes = 3;
- required uint32 entryPoint = 4;
- required tvm_keys.BbEnvType env = 5;
-}
-
-message ServiceTicket {
- required uint32 srcClientId = 1;
- required uint32 dstClientId = 2;
- repeated string scopes = 3;
- optional uint64 issuerUid = 4;
-}
-
-message Ticket {
- required uint32 keyId = 1;
- required int64 expirationTime = 2;
- optional UserTicket user = 3;
- optional ServiceTicket service = 4;
-}
diff --git a/library/cpp/tvmauth/src/protos/tvm_keys.proto b/library/cpp/tvmauth/src/protos/tvm_keys.proto
deleted file mode 100644
index 9ba42dbf80..0000000000
--- a/library/cpp/tvmauth/src/protos/tvm_keys.proto
+++ /dev/null
@@ -1,36 +0,0 @@
-package tvm_keys;
-
-option go_package = "a.yandex-team.ru/library/cpp/tvmauth/src/protos";
-
-enum KeyType {
- RabinWilliams = 0;
-}
-
-enum BbEnvType {
- Prod = 0;
- Test = 1;
- ProdYateam = 2;
- TestYateam = 3;
- Stress = 4;
-}
-
-message General {
- required uint32 id = 1;
- required KeyType type = 2;
- required bytes body = 3;
- optional int64 createdTime = 4;
-}
-
-message BbKey {
- required General gen = 1;
- required BbEnvType env = 2;
-}
-
-message TvmKey {
- required General gen = 1;
-}
-
-message Keys {
- repeated BbKey bb = 1;
- repeated TvmKey tvm = 2;
-}
diff --git a/library/cpp/tvmauth/src/rw/CMakeLists.txt b/library/cpp/tvmauth/src/rw/CMakeLists.txt
deleted file mode 100644
index 8b37918efe..0000000000
--- a/library/cpp/tvmauth/src/rw/CMakeLists.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-
-# This file was gererated by the build system used internally in the Yandex monorepo.
-# Only simple modifications are allowed (adding source-files to targets, adding simple properties
-# like target_include_directories). These modifications will be ported to original
-# ya.make files by maintainers. Any complex modifications which can't be ported back to the
-# original buildsystem will not be accepted.
-
-
-find_package(OpenSSL REQUIRED)
-
-add_library(ticket_parser)
-target_link_libraries(ticket_parser PUBLIC
- contrib-libs-cxxsupp
- yutil
- OpenSSL::OpenSSL
- cpp-openssl-init
-)
-target_sources(ticket_parser PRIVATE
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/keys.cpp
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_asn1.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_key.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_lib.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_ossl.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_pss.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_pss_sign.c
- ${CMAKE_SOURCE_DIR}/library/cpp/tvmauth/src/rw/rw_sign.c
-)
diff --git a/library/cpp/tvmauth/src/rw/keys.cpp b/library/cpp/tvmauth/src/rw/keys.cpp
deleted file mode 100644
index 5395287f5c..0000000000
--- a/library/cpp/tvmauth/src/rw/keys.cpp
+++ /dev/null
@@ -1,138 +0,0 @@
-#include "keys.h"
-
-#include "rw.h"
-
-#include <library/cpp/openssl/init/init.h>
-
-#include <contrib/libs/openssl/include/openssl/evp.h>
-
-#include <util/generic/strbuf.h>
-#include <util/generic/yexception.h>
-
-namespace {
- struct TInit {
- TInit() {
- InitOpenSSL();
- }
- } INIT;
-}
-
-namespace NTvmAuth {
- namespace NRw {
- namespace NPrivate {
- void TRwDestroyer::Destroy(TRwInternal* o) {
- RwFree(o);
- }
-
- class TArrayDestroyer {
- public:
- static void Destroy(unsigned char* o) {
- free(o);
- }
- };
- }
-
- static TString SerializeRW(TRwKey* rw, int (*func)(const TRwKey*, unsigned char**)) {
- unsigned char* buf = nullptr;
- int size = func(rw, &buf);
- THolder<unsigned char, NPrivate::TArrayDestroyer> guard(buf);
- return TString((char*)buf, size);
- }
-
- TKeyPair GenKeyPair(size_t size) {
- TRw rw(RwNew());
- RwGenerateKey(rw.Get(), size);
-
- TRw skey(RwPrivateKeyDup(rw.Get()));
- TRw vkey(RwPublicKeyDup(rw.Get()));
-
- TKeyPair res;
- res.Private = SerializeRW(skey.Get(), &i2d_RWPrivateKey);
- res.Public = SerializeRW(vkey.Get(), &i2d_RWPublicKey);
-
- TRwPrivateKey prKey(res.Private, 0);
- TRwPublicKey pubKey(res.Public);
-
- const TStringBuf msg = "Test test test test test";
-
- Y_ENSURE(pubKey.CheckSign(msg, prKey.SignTicket(msg)), "Failed to gen keys");
-
- return res;
- }
-
- TRwPrivateKey::TRwPrivateKey(TStringBuf body, TKeyId id)
- : Id_(id)
- , Rw_(Deserialize(body))
- , SignLen_(RwModSize(Rw_.Get()))
- {
- Y_ENSURE(SignLen_ > 0, "Private key has bad len: " << SignLen_);
- }
-
- TKeyId TRwPrivateKey::GetId() const {
- return Id_;
- }
-
- TString TRwPrivateKey::SignTicket(TStringBuf ticket) const {
- TString res(SignLen_, 0x00);
-
- int len = RwPssrSignMsg(ticket.size(),
- (const unsigned char*)ticket.data(),
- (unsigned char*)res.data(),
- Rw_.Get(),
- (EVP_MD*)EVP_sha256());
-
- Y_ENSURE(len > 0 && len <= SignLen_, "Signing failed. len: " << len);
-
- res.resize(len);
- return res;
- }
-
- TRw TRwPrivateKey::Deserialize(TStringBuf key) {
- TRwKey* rw = nullptr;
- auto data = reinterpret_cast<const unsigned char*>(key.data());
- if (!d2i_RWPrivateKey(&rw, &data, key.size())) {
- ythrow yexception() << "Private key is malformed";
- }
- return TRw(rw);
- }
-
- TRwPublicKey::TRwPublicKey(TStringBuf body)
- : Rw_(Deserialize(body))
- {
- }
-
- bool TRwPublicKey::CheckSign(TStringBuf ticket, TStringBuf sign) const {
- int result = RwPssrVerifyMsg(ticket.size(),
- (const unsigned char*)ticket.data(),
- (unsigned char*)sign.data(),
- sign.size(),
- Rw_.Get(),
- (EVP_MD*)EVP_sha256());
-
- Y_ENSURE(result >= 0, "Failed to check sign: " << result);
- return result;
- }
-
- TRw TRwPublicKey::Deserialize(TStringBuf key) {
- TRwKey* rw = nullptr;
- auto data = reinterpret_cast<const unsigned char*>(key.data());
- auto status = d2i_RWPublicKey(&rw, &data, key.size());
-
- TRw res(rw);
- Y_ENSURE(status, "Public key is malformed: " << key);
- return res;
- }
-
- TSecureHeap::TSecureHeap(size_t totalSize, int minChunkSize) {
- CRYPTO_secure_malloc_init(totalSize, minChunkSize);
- }
-
- TSecureHeap::~TSecureHeap() {
- CRYPTO_secure_malloc_done();
- }
-
- void TSecureHeap::Init(size_t totalSize, int minChunkSize) {
- Singleton<TSecureHeap>(totalSize, minChunkSize);
- }
- }
-}
diff --git a/library/cpp/tvmauth/src/rw/keys.h b/library/cpp/tvmauth/src/rw/keys.h
deleted file mode 100644
index e02b7e72a1..0000000000
--- a/library/cpp/tvmauth/src/rw/keys.h
+++ /dev/null
@@ -1,65 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-
-#include <unordered_map>
-
-struct TRwInternal;
-
-namespace NTvmAuth {
- namespace NRw {
- namespace NPrivate {
- class TRwDestroyer {
- public:
- static void Destroy(TRwInternal* o);
- };
- }
-
- using TRw = THolder<TRwInternal, NPrivate::TRwDestroyer>;
- using TKeyId = ui32;
-
- struct TKeyPair {
- TString Private;
- TString Public;
- };
- TKeyPair GenKeyPair(size_t size);
-
- class TRwPrivateKey {
- public:
- TRwPrivateKey(TStringBuf body, TKeyId id);
-
- TKeyId GetId() const;
- TString SignTicket(TStringBuf ticket) const;
-
- private:
- static TRw Deserialize(TStringBuf key);
-
- TKeyId Id_;
- TRw Rw_;
- int SignLen_;
- };
-
- class TRwPublicKey {
- public:
- TRwPublicKey(TStringBuf body);
-
- bool CheckSign(TStringBuf ticket, TStringBuf sign) const;
-
- private:
- static TRw Deserialize(TStringBuf key);
-
- TRw Rw_;
- };
-
- using TPublicKeys = std::unordered_map<TKeyId, TRwPublicKey>;
-
- class TSecureHeap {
- public:
- TSecureHeap(size_t totalSize, int minChunkSize);
- ~TSecureHeap();
-
- static void Init(size_t totalSize = 16 * 1024 * 1024, int minChunkSize = 16);
- };
- }
-}
diff --git a/library/cpp/tvmauth/src/rw/rw.h b/library/cpp/tvmauth/src/rw/rw.h
deleted file mode 100644
index cbff96b85d..0000000000
--- a/library/cpp/tvmauth/src/rw/rw.h
+++ /dev/null
@@ -1,86 +0,0 @@
-#pragma once
-
-#include <contrib/libs/openssl/include/openssl/bn.h>
-#include <contrib/libs/openssl/include/openssl/crypto.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
- typedef struct {
- BIGNUM* S;
- } TRwSignature;
-
- /*Rabin–Williams*/
- typedef struct TRwInternal TRwKey;
-
- typedef struct {
- TRwSignature* (*RwSign)(const unsigned char* dgst, const int dlen, TRwKey* rw);
- int (*RwVerify)(const unsigned char* dgst, int dgst_len, TRwSignature* sig, const TRwKey* rw);
- int (*RwApply)(BIGNUM* r, BIGNUM* x, BN_CTX* ctx, const TRwKey* rw);
- } TRwMethod;
-
- struct TRwInternal {
- /* first private multiplier */
- BIGNUM* P;
- /* second private multiplier */
- BIGNUM* Q;
- /* n = p*q - RW modulus */
- BIGNUM* N;
- /* precomputed 2^((3q-5)/8) mod q */
- BIGNUM* Twomq;
- /* precomputed 2^((9p-11)/8) mod p*/
- BIGNUM* Twomp;
- /* precomputed q^(p-2) == q^(-1) mod p */
- BIGNUM* Iqmp;
- /* (q+1) / 8 */
- BIGNUM* Dq;
- /* (p-3) / 8 */
- BIGNUM* Dp;
- /* functions for working with RW */
- const TRwMethod* Meth;
- };
-
- TRwSignature* RwSignatureNew(void);
- void RwSignatureFree(TRwSignature* a);
-
- /* RW signing functions */
- /* the function can put some tmp values to rw */
- int RwPssrSignHash(const unsigned char* from, unsigned char* to, TRwKey* rw, const EVP_MD* md);
- int RwPssrSignMsg(const int msgLen, const unsigned char* msg, unsigned char* to, TRwKey* rw, const EVP_MD* md);
-
- /* RW-PSS verification functions */
- int RwPssrVerifyHash(const unsigned char* from, const unsigned char* sig, const int sig_len, const TRwKey* rw, const EVP_MD* md);
- int RwPssrVerifyMsg(const int msgLen, const unsigned char* msg, const unsigned char* sig, const int sig_len, const TRwKey* rw, const EVP_MD* md);
-
- /* internal functions, use them only if you know what you're doing */
- int RwNoPaddingSign(int flen, const unsigned char* from, unsigned char* to, TRwKey* rw);
- int RwApply(const int flen, const unsigned char* from, unsigned char* to, const TRwKey* rw);
-
- const TRwMethod* RwDefaultMethods(void);
-
- TRwKey* RwNew(void);
- void RwFree(TRwKey* r);
- int RwSize(const TRwKey* rw);
- int RwModSize(const TRwKey* rw);
-
- TRwKey* RwPublicKeyDup(TRwKey* rw);
- TRwKey* RwPrivateKeyDup(TRwKey* rw);
-
- // NOLINTNEXTLINE(readability-identifier-naming)
- TRwKey* d2i_RWPublicKey(TRwKey** a, const unsigned char** pp, long length);
- // NOLINTNEXTLINE(readability-identifier-naming)
- TRwKey* d2i_RWPrivateKey(TRwKey** a, const unsigned char** pp, long length);
-
- int RwGenerateKey(TRwKey* a, int bits);
- // NOLINTNEXTLINE(readability-identifier-naming)
- int i2d_RWPublicKey(const TRwKey* a, unsigned char** pp);
- // NOLINTNEXTLINE(readability-identifier-naming)
- int i2d_RWPrivateKey(const TRwKey* a, unsigned char** pp);
-
- int RwPaddingAddPssr(const TRwKey* rw, unsigned char* EM, const unsigned char* mHash, const EVP_MD* Hash, int sLen);
- int RwVerifyPssr(const TRwKey* rw, const unsigned char* mHash, const EVP_MD* Hash, const unsigned char* EM, int sLen);
-
-#ifdef __cplusplus
-}
-#endif
diff --git a/library/cpp/tvmauth/src/rw/rw_asn1.c b/library/cpp/tvmauth/src/rw/rw_asn1.c
deleted file mode 100644
index 76682dcff4..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_asn1.c
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/asn1.h>
-#include <contrib/libs/openssl/include/openssl/asn1t.h>
-#include <contrib/libs/openssl/include/openssl/rand.h>
-
-#include <stdio.h>
-
-/* Override the default new methods */
-/* This callback is used by OpenSSL's ASN.1 parser */
-static int SignatureCallback(int operation, ASN1_VALUE** pval, const ASN1_ITEM* it, void* exarg) {
- (void)it;
- (void)exarg;
-
- if (operation == ASN1_OP_NEW_PRE) {
- TRwSignature* sig;
- sig = OPENSSL_malloc(sizeof(TRwSignature));
- if (!sig)
- return 0;
- sig->S = NULL;
- *pval = (ASN1_VALUE*)sig;
- return 2;
- }
- return 1;
-}
-
-/* ASN.1 structure representing RW signature value */
-ASN1_SEQUENCE_cb(TRwSignature, SignatureCallback) = {
- ASN1_SIMPLE(TRwSignature, S, BIGNUM),
-} ASN1_SEQUENCE_END_cb(TRwSignature, TRwSignature)
-
- /* i2d_ and d2i functions implementation for RW */
- IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(TRwSignature, TRwSignature, TRwSignature)
-
- /* Override the default free and new methods */
- static int RwCallback(int operation, ASN1_VALUE** pval, const ASN1_ITEM* it, void* exarg) {
- (void)it;
- (void)exarg;
-
- if (operation == ASN1_OP_NEW_PRE) {
- *pval = (ASN1_VALUE*)RwNew();
- if (*pval)
- return 2;
- return 0;
- } else if (operation == ASN1_OP_FREE_PRE) {
- RwFree((TRwKey*)*pval);
- *pval = NULL;
- return 2;
- }
- return 1;
-}
-
-/* ASN.1 representation of RW's private key */
-ASN1_SEQUENCE_cb(RWPrivateKey, RwCallback) = {
- ASN1_SIMPLE(TRwKey, N, BIGNUM),
- ASN1_SIMPLE(TRwKey, P, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Q, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Iqmp, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Dq, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Dp, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Twomp, CBIGNUM),
- ASN1_SIMPLE(TRwKey, Twomq, CBIGNUM)} ASN1_SEQUENCE_END_cb(TRwKey, RWPrivateKey);
-
-/* i2d_ and d2i_ functions for RW's private key */
-IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(TRwKey, RWPrivateKey, RWPrivateKey);
-
-/* ASN.1 representation of RW public key */
-ASN1_SEQUENCE_cb(RWPublicKey, RwCallback) = {
- ASN1_SIMPLE(TRwKey, N, BIGNUM),
-} ASN1_SEQUENCE_END_cb(TRwKey, RWPublicKey);
-
-/* i2d_ and d2i functions for RW public key */
-IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(TRwKey, RWPublicKey, RWPublicKey);
-
-TRwKey* RwPublicKeyDup(TRwKey* rw) {
- return ASN1_item_dup(ASN1_ITEM_rptr(RWPublicKey), rw);
-}
-
-TRwKey* RwPrivateKeyDup(TRwKey* rw) {
- return ASN1_item_dup(ASN1_ITEM_rptr(RWPrivateKey), rw);
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_key.c b/library/cpp/tvmauth/src/rw/rw_key.c
deleted file mode 100644
index 8375c3ca20..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_key.c
+++ /dev/null
@@ -1,135 +0,0 @@
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/rand.h>
-
-int RwGenerateKey(TRwKey* rw, int bits) {
- int ok = 0;
-
- BN_CTX* ctx = NULL;
- BIGNUM *rem3 = NULL, *rem7 = NULL, *mod8 = NULL, *rem5 = NULL;
- BIGNUM *nmod = NULL, *twomqexp = NULL, *twompexp = NULL, *two = NULL;
-
- int bitsp = (bits + 1) / 2;
- int bitsq = bits - bitsp;
-
- /* make sure that all components are not null */
- if ((ctx = BN_CTX_secure_new()) == NULL)
- goto err;
- if (!rw)
- goto err;
- if (!rw->N && ((rw->N = BN_new()) == NULL))
- goto err;
- if (!rw->P && ((rw->P = BN_new()) == NULL))
- goto err;
- if (!rw->Q && ((rw->Q = BN_new()) == NULL))
- goto err;
- if (!rw->Iqmp && ((rw->Iqmp = BN_new()) == NULL))
- goto err;
- if (!rw->Twomq && ((rw->Twomq = BN_new()) == NULL))
- goto err;
- if (!rw->Twomp && ((rw->Twomp = BN_new()) == NULL))
- goto err;
- if (!rw->Dq && ((rw->Dq = BN_new()) == NULL))
- goto err;
- if (!rw->Dp && ((rw->Dp = BN_new()) == NULL))
- goto err;
-
- BN_CTX_start(ctx);
-
- rem3 = BN_CTX_get(ctx);
- rem7 = BN_CTX_get(ctx);
- rem5 = BN_CTX_get(ctx);
- mod8 = BN_CTX_get(ctx);
- nmod = BN_CTX_get(ctx);
- twomqexp = BN_CTX_get(ctx);
- twompexp = BN_CTX_get(ctx);
- two = BN_CTX_get(ctx);
-
- if (!BN_set_word(mod8, 8))
- goto err;
- if (!BN_set_word(rem3, 3))
- goto err;
- if (!BN_set_word(rem7, 7))
- goto err;
- if (!BN_set_word(rem5, 5))
- goto err;
- if (!BN_set_word(two, 2))
- goto err;
-
- /* generate p */
- /* add == 8 */
- /* rem == 3 */
- /* safe == 0 as we don't need (p-1)/2 to be also prime */
- if (!BN_generate_prime_ex(rw->P, bitsp, 0, mod8, rem3, NULL))
- goto err;
-
- /* generate q */
- /* add == 8 */
- /* rem == 7 */
- /* safe == 0 */
- if (!BN_generate_prime_ex(rw->Q, bitsq, 0, mod8, rem7, NULL))
- goto err;
-
- /* n == p*q */
- if (!BN_mul(rw->N, rw->P, rw->Q, ctx))
- goto err;
-
- /* n == 5 mod 8 ? */
- if (!BN_nnmod(nmod, rw->N, mod8, ctx))
- goto err;
- if (BN_ucmp(rem5, nmod) != 0)
- goto err;
-
- /* q^(-1) mod p */
- if (!BN_mod_inverse(rw->Iqmp, rw->Q, rw->P, ctx))
- goto err;
-
- /* twomqexp = (3q-5)/8 */
- if (!BN_copy(twomqexp, rw->Q))
- goto err;
- if (!BN_mul_word(twomqexp, 3))
- goto err;
- if (!BN_sub_word(twomqexp, 5))
- goto err;
- if (!BN_rshift(twomqexp, twomqexp, 3))
- goto err;
- if (!BN_mod_exp(rw->Twomq, two, twomqexp, rw->Q, ctx))
- goto err;
-
- /* twompexp = (9p-11)/8 */
- if (!BN_copy(twompexp, rw->P))
- goto err;
- if (!BN_mul_word(twompexp, 9))
- goto err;
- if (!BN_sub_word(twompexp, 11))
- goto err;
- if (!BN_rshift(twompexp, twompexp, 3))
- goto err;
- if (!BN_mod_exp(rw->Twomp, two, twompexp, rw->P, ctx))
- goto err;
-
- /* dp = (p-3) / 8 */
- if (!BN_copy(rw->Dp, rw->P))
- goto err;
- if (!BN_sub_word(rw->Dp, 3))
- goto err;
- if (!BN_rshift(rw->Dp, rw->Dp, 3))
- goto err;
-
- /* dq = (q+1) / 8 */
- if (!BN_copy(rw->Dq, rw->Q))
- goto err;
- if (!BN_add_word(rw->Dq, 1))
- goto err;
- if (!BN_rshift(rw->Dq, rw->Dq, 3))
- goto err;
-
- ok = 1;
-
-err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
- return ok;
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_lib.c b/library/cpp/tvmauth/src/rw/rw_lib.c
deleted file mode 100644
index 94d94caa4a..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_lib.c
+++ /dev/null
@@ -1,77 +0,0 @@
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/asn1.h>
-
-#include <stdio.h>
-
-TRwKey* RwNew(void) {
- TRwKey* ret = NULL;
-
- ret = (TRwKey*)malloc(sizeof(TRwKey));
- if (ret == NULL) {
- return (NULL);
- }
- ret->Meth = RwDefaultMethods();
-
- ret->P = NULL;
- ret->Q = NULL;
- ret->N = NULL;
- ret->Iqmp = NULL;
- ret->Twomq = NULL;
- ret->Twomp = NULL;
- ret->Dp = NULL;
- ret->Dq = NULL;
-
- return ret;
-}
-
-void RwFree(TRwKey* r) {
- if (r == NULL)
- return;
-
- if (r->P != NULL)
- BN_clear_free(r->P);
- if (r->Q != NULL)
- BN_clear_free(r->Q);
- if (r->N != NULL)
- BN_clear_free(r->N);
- if (r->Iqmp != NULL)
- BN_clear_free(r->Iqmp);
- if (r->Dp != NULL)
- BN_clear_free(r->Dp);
- if (r->Dq != NULL)
- BN_clear_free(r->Dq);
- if (r->Twomp != NULL)
- BN_clear_free(r->Twomp);
- if (r->Twomq != NULL)
- BN_clear_free(r->Twomq);
-
- free(r);
-}
-
-int RwSize(const TRwKey* r) {
- int ret = 0, i = 0;
- ASN1_INTEGER bs;
- unsigned char buf[4]; /* 4 bytes looks really small.
- However, i2d_ASN1_INTEGER() will not look
- beyond the first byte, as long as the second
- parameter is NULL. */
-
- i = BN_num_bits(r->N);
- bs.length = (i + 7) / 8;
- bs.data = buf;
- bs.type = V_ASN1_INTEGER;
- /* If the top bit is set the asn1 encoding is 1 larger. */
- buf[0] = 0xff;
-
- i = i2d_ASN1_INTEGER(&bs, NULL);
-
- ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);
- return ret;
-}
-
-int RwModSize(const TRwKey* rw) {
- if (rw == NULL || rw->N == NULL)
- return 0;
- return BN_num_bytes(rw->N);
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_ossl.c b/library/cpp/tvmauth/src/rw/rw_ossl.c
deleted file mode 100644
index 951752bdb3..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_ossl.c
+++ /dev/null
@@ -1,473 +0,0 @@
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/rand.h>
-
-//#define RW_PRINT_DEBUG
-//#define AVOID_IF
-//#define FAULT_TOLERANCE_CHECK
-
-#ifdef RW_PRINT_DEBUG
- #include <stdio.h>
-#endif
-
-static TRwSignature* RwDoSign(const unsigned char* dgst, int dlen, TRwKey* rw);
-static int RwDoVerify(const unsigned char* dgst, int dgst_len, TRwSignature* sig, const TRwKey* rw);
-static int RwDoApply(BIGNUM* r, BIGNUM* x, BN_CTX* ctx, const TRwKey* rw);
-
-static TRwMethod rw_default_meth = {
- RwDoSign,
- RwDoVerify,
- RwDoApply};
-
-const TRwMethod* RwDefaultMethods(void) {
- return &rw_default_meth;
-}
-
-#ifdef RW_PRINT_DEBUG
-
-static void print_bn(char* name, BIGNUM* value) {
- char* str_repr;
- str_repr = BN_bn2dec(value);
- printf("Name: %s\n", name);
- printf("Value: %s\n", str_repr);
- OPENSSL_free(str_repr);
-}
-
- #define DEBUG_PRINT_BN(s, x) \
- do { \
- print_bn((s), (x)); \
- } while (0);
- #define DEBUG_PRINT_RW(r) \
- do { \
- DEBUG_PRINT_BN("rw->p", (r)->p); \
- DEBUG_PRINT_BN("rw->q", (r)->q); \
- DEBUG_PRINT_BN("rw->n", (r)->n); \
- DEBUG_PRINT_BN("rw->iqmp", (r)->iqmp); \
- DEBUG_PRINT_BN("rw->twomp", (r)->twomp); \
- DEBUG_PRINT_BN("rw->twomq", (r)->twomq); \
- DEBUG_PRINT_BN("rw->dp", (r)->dp); \
- DEBUG_PRINT_BN("rw->dq", (r)->dq); \
- } while (0);
- #define DEBUG_PRINTF(s, v) \
- do { \
- printf((s), (v)); \
- } while (0);
-#else
- #define DEBUG_PRINT_BN(s, x)
- #define DEBUG_PRINT_RW(r)
- #define DEBUG_PRINTF(s, v)
-#endif
-
-/*
- * The algorithms was taken from
- * https://cr.yp.to/sigs/rwsota-20080131.pdf
- * Section 6 -> "Avoiding Jacobi symbols"
- * '^' means power
- * 1. Compute U = h ^ ((q+1) / 8) mod q
- * 2. If U ^ 4 - h mod q == 0, set e = 1 otherwise set e = -1
- * 3. Compute V = (eh) ^ ((p-3)/8) mod p
- * 4. If (V^4 * (eh)^2 - eh) mod p = 0; set f = 1; otherwise set f = 2
- * 5. Precompute 2^((3q-5) / 8) mod q; Compute W = f^((3*q - 5) / 8) * U mod q
- * 6. Precompute 2^((9p-11) / 8) mod p; Compute X = f^((9p-11) / 8) * V^3 * eh mod p
- * 7. Precompute q^(p-2) mod p; Compute Y = W + q(q^(p-2) * (X - W) mod p)
- * 8. Compute s = Y^2 mod pq
- * 9. Fault tolerance: if efs^2 mod pq != h start over
- */
-static TRwSignature* RwDoSign(const unsigned char* dgst, int dlen, TRwKey* rw) {
- BIGNUM *m, *U, *V, *tmp, *m_q, *m_p, *tmp2;
- /* additional variables to avoid "if" statements */
- BIGNUM *tmp_mp, *tmp_U, *tmp_V;
- TRwSignature* ret = NULL;
- BN_CTX* ctx = NULL;
- int ok = 0, e = 0, f = 0;
-
- if (!rw || !rw->P || !rw->Q || !rw->N || !rw->Iqmp || !rw->Dp || !rw->Dq || !rw->Twomp || !rw->Twomq)
- goto err;
-
- if ((ctx = BN_CTX_secure_new()) == NULL)
- goto err;
- BN_CTX_start(ctx);
-
- m = BN_CTX_get(ctx);
- U = BN_CTX_get(ctx);
- V = BN_CTX_get(ctx);
- tmp = BN_CTX_get(ctx);
- tmp2 = BN_CTX_get(ctx);
- m_q = BN_CTX_get(ctx);
- m_p = BN_CTX_get(ctx);
- tmp_mp = BN_CTX_get(ctx);
- tmp_U = BN_CTX_get(ctx);
- tmp_V = BN_CTX_get(ctx);
-
- DEBUG_PRINT_RW(rw)
-
- /* if (!BN_set_word(four, 4)) goto err; */
-
- if (!BN_bin2bn(dgst, dlen, m))
- goto err;
- if (BN_ucmp(m, rw->N) >= 0)
- goto err;
-
- /* check if m % 16 == 12 */
- if (BN_mod_word(m, 16) != 12)
- goto err;
- DEBUG_PRINT_BN("m", m)
-
- /* TODO: optimization to avoid memory allocation? */
- if ((ret = RwSignatureNew()) == NULL)
- goto err;
- /* memory allocation */
- if ((ret->S = BN_new()) == NULL)
- goto err;
-
- /* m_q = m mod q */
- if (!BN_nnmod(m_q, m, rw->Q, ctx))
- goto err;
- /* m_p = m mod p */
- if (!BN_nnmod(m_p, m, rw->P, ctx))
- goto err;
-
- DEBUG_PRINT_BN("m_p", m_p)
- DEBUG_PRINT_BN("m_q", m_q)
-
- /* U = h ** ((q+1)/8) mod q */
- if (!BN_mod_exp(U, m_q, rw->Dq, rw->Q, ctx))
- goto err;
- DEBUG_PRINT_BN("U", U)
-
- /* tmp = U^4 - h mod q */
- if (!BN_mod_sqr(tmp, U, rw->Q, ctx))
- goto err;
- if (!BN_mod_sqr(tmp, tmp, rw->Q, ctx))
- goto err;
- DEBUG_PRINT_BN("U**4 mod q", tmp)
-
- /* e = 1 if tmp == 0 else -1 */
- e = 2 * (BN_ucmp(tmp, m_q) == 0) - 1;
- DEBUG_PRINTF("e == %i\n", e)
-
- /*
- to avoid "if" branch
- if e == -1: m_p = tmp_mp
- if e == 1: m_p = m_p
- */
- if (!BN_sub(tmp_mp, rw->P, m_p))
- goto err;
- m_p = (BIGNUM*)((1 - ((1 + e) >> 1)) * (BN_ULONG)tmp_mp + ((1 + e) >> 1) * (BN_ULONG)m_p);
- DEBUG_PRINT_BN("eh mod p", m_p)
-
- /* V = (eh) ** ((p-3)/8) */
- if (!BN_mod_exp(V, m_p, rw->Dp, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("V == ((eh) ** ((p-3)/8))", V)
-
- /* (eh) ** 2 */
- if (!BN_mod_sqr(tmp2, m_p, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("(eh)**2", tmp2)
-
- /* V ** 4 */
- if (!BN_mod_sqr(tmp, V, rw->P, ctx))
- goto err;
- if (!BN_mod_sqr(tmp, tmp, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("V**4", tmp)
-
- /* V**4 * (eh)**2 */
- if (!BN_mod_mul(tmp, tmp, tmp2, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("tmp = (V**4 * (eh)**2) mod p", tmp)
-
- /* tmp = tmp - eh mod p */
- if (!BN_mod_sub(tmp, tmp, m_p, rw->P, ctx))
- goto err;
-
- /* f = 1 if zero else 2 */
- f = 2 - BN_is_zero(tmp);
- /* f = 2 - (constant_time_is_zero(BN_ucmp(tmp, m_p)) & 1); */
- DEBUG_PRINTF("f == %i\n", f)
-
-#ifdef AVOID_IF
- if (!BN_mod_mul(tmp_U, U, rw->twomq, rw->q, ctx))
- goto err;
-
- /*
- to avoid "if" branch we use tiny additional computation
- */
- U = (BIGNUM*)((2 - f) * (BN_ULONG)U + (1 - (2 - f)) * (BN_ULONG)tmp_U);
-#else
-
- if (f == 2) {
- if (!BN_mod_mul(U, U, rw->Twomq, rw->Q, ctx))
- goto err;
- }
-
-#endif
-
- DEBUG_PRINT_BN("W", U)
-
- /* V ** 3 */
- if (!BN_mod_sqr(tmp, V, rw->P, ctx))
- goto err;
- if (!BN_mod_mul(V, V, tmp, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("V**3", V)
-
- /* *(eh) */
- if (!BN_mod_mul(V, V, m_p, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("V**3 * (eh) mod p", V)
-
-#ifdef AVOID_IF
-
- /* to avoid "if" statement we use simple computation */
- if (!BN_mod_mul(tmp_V, V, rw->twomp, rw->p, ctx))
- goto err;
- V = (BIGNUM*)((2 - f) * (BN_ULONG)V + (1 - (2 - f)) * (BN_ULONG)tmp_V);
-
-#else
-
- if (f == 2) {
- if (!BN_mod_mul(V, V, rw->Twomp, rw->P, ctx))
- goto err;
- }
-
-#endif
-
- DEBUG_PRINT_BN("X", V)
-
- /* W = U, X = V */
- if (!BN_mod_sub(V, V, U, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("X - W mod p", V)
-
- if (!BN_mod_mul(V, V, rw->Iqmp, rw->P, ctx))
- goto err;
- DEBUG_PRINT_BN("q**(p-2) * (X-W) mod p", V)
-
- if (!BN_mul(V, V, rw->Q, ctx))
- goto err;
- DEBUG_PRINT_BN("q * prev mod p", V)
-
- if (!BN_mod_add(V, U, V, rw->N, ctx))
- goto err;
- DEBUG_PRINT_BN("Y", V)
-
- /* now V = Y */
- if (!BN_mod_sqr(V, V, rw->N, ctx))
- goto err;
- DEBUG_PRINT_BN("s", V)
-
-#ifdef FAULT_TOLERANCE_CHECK
-
- /* now V = s - principal square root */
- /* fault tolerance check */
- if (!BN_mod_sqr(tmp, V, rw->n, ctx))
- goto err;
- DEBUG_PRINT_BN("s**2", tmp)
-
- if (!BN_mul_word(tmp, f))
- goto err;
- DEBUG_PRINT_BN("f * s**2", tmp)
-
- if (!BN_nnmod(tmp, tmp, rw->n, ctx))
- goto err;
- DEBUG_PRINT_BN("s**2 * f mod n", tmp)
-
- /* to avoid "if" statement */
- if (!BN_sub(tmp2, rw->n, tmp))
- goto err;
- tmp = (BIGNUM*)(((1 + e) >> 1) * (BN_ULONG)tmp + (1 - ((1 + e) >> 1)) * (BN_ULONG)tmp2);
- DEBUG_PRINT_BN("ef(s**2)", tmp)
- DEBUG_PRINT_BN("(tmp == original m)", tmp)
-
- if (BN_ucmp(tmp, m) != 0)
- goto err;
-
-#endif
-
- /* making the "principal square root" to be "|principal| square root" */
- if (!BN_sub(tmp, rw->N, V))
- goto err;
-
- /* if tmp = MIN(V, rw->n - V) */
- tmp = BN_ucmp(tmp, V) >= 0 ? V : tmp;
-
- if (!BN_copy(ret->S, tmp))
- goto err;
-
- ok = 1;
-
-err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
- if (!ok) {
- RwSignatureFree(ret);
- ret = NULL;
- }
-
- return ret;
-}
-
-static int RwDoVerify(const unsigned char* dgst, int dgst_len, TRwSignature* sig, const TRwKey* rw) {
- BIGNUM *m = NULL, *x = NULL, *t1 = NULL, *t2 = NULL, *t1d = NULL, *t2d = NULL;
- BN_CTX* ctx = NULL;
- BN_ULONG rest1 = 0, rest2 = 0;
- int retval = 0;
-
- if (!rw || !rw->N || !sig || !sig->S)
- goto err;
-
- if ((ctx = BN_CTX_secure_new()) == NULL)
- goto err;
- BN_CTX_start(ctx);
-
- m = BN_CTX_get(ctx);
- t1 = BN_CTX_get(ctx);
- t2 = BN_CTX_get(ctx);
- t1d = BN_CTX_get(ctx);
- t2d = BN_CTX_get(ctx);
-
- if (!BN_bin2bn(dgst, dgst_len, m))
- goto err;
- /* dgst too big */
- if (!BN_copy(t1, rw->N))
- goto err;
- if (!BN_sub_word(t1, 1))
- goto err;
- if (!BN_rshift(t1, t1, 1))
- goto err;
-
- /* check m and rw->n relation */
- if (BN_ucmp(m, rw->N) >= 0)
- goto err;
- rest1 = BN_mod_word(m, 16);
- if (rest1 != 12)
- goto err;
-
- if (BN_ucmp(t1, sig->S) < 0)
- goto err;
- if (BN_is_negative(sig->S))
- goto err;
-
- if (!BN_mod_sqr(t1, sig->S, rw->N, ctx))
- goto err;
- if (!BN_sub(t2, rw->N, t1))
- goto err;
- if (!BN_lshift1(t1d, t1))
- goto err;
- if (!BN_lshift1(t2d, t2))
- goto err;
-
- rest1 = BN_mod_word(t1, 16);
- rest2 = BN_mod_word(t2, 16);
-
- /* mod 16 */
- if (rest1 == 12) {
- x = t1;
- }
- /* mod 8 */
- else if ((rest1 & 0x07) == 6) {
- x = t1d;
- }
- /* mod 16 */
- else if (rest2 == 12) {
- x = t2;
- }
- /* mod 8 */
- else if ((rest2 & 0x07) == 6) {
- x = t2d;
- } else
- goto err;
-
- DEBUG_PRINT_BN("m", m)
- DEBUG_PRINT_BN("x", x)
-
- /* check signature value */
- retval = BN_ucmp(m, x) == 0;
-
-err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
- return retval;
-}
-
-static int RwDoApply(BIGNUM* r, BIGNUM* x, BN_CTX* ctx, const TRwKey* rw) {
- BIGNUM *t1 = NULL, *t2 = NULL, *t1d = NULL, *t2d = NULL, *rs = NULL;
- BN_ULONG rest1 = 0, rest2 = 0;
- int retval = 0;
-
- if (!rw || !rw->N || !x || !ctx || !r)
- goto err;
-
- DEBUG_PRINT_BN("Signature = x = ", x)
- DEBUG_PRINT_BN("n", rw->n)
-
- BN_CTX_start(ctx);
-
- t1 = BN_CTX_get(ctx);
- t2 = BN_CTX_get(ctx);
- t1d = BN_CTX_get(ctx);
- t2d = BN_CTX_get(ctx);
-
- if (!BN_copy(t1, rw->N))
- goto err;
- if (!BN_sub_word(t1, 1))
- goto err;
- if (!BN_rshift(t1, t1, 1))
- goto err;
-
- /* check m and rw->n relation */
- if (BN_ucmp(x, rw->N) >= 0)
- goto err;
-
- if (BN_ucmp(t1, x) < 0)
- goto err;
- if (BN_is_negative(x))
- goto err;
-
- if (!BN_mod_sqr(t1, x, rw->N, ctx))
- goto err;
- DEBUG_PRINT_BN("x**2 mod n", t1)
-
- if (!BN_sub(t2, rw->N, t1))
- goto err;
- DEBUG_PRINT_BN("n - x**2", t2)
-
- if (!BN_lshift1(t1d, t1))
- goto err;
- if (!BN_lshift1(t2d, t2))
- goto err;
-
- rest1 = BN_mod_word(t1, 16);
- rest2 = BN_mod_word(t2, 16);
-
- /* mod 16 */
- if (rest1 == 12) {
- rs = t1;
- }
- /* mod 8 */
- else if ((rest1 & 0x07) == 6) {
- rs = t1d;
- }
- /* mod 16 */
- else if (rest2 == 12) {
- rs = t2;
- }
- /* mod 8 */
- else if ((rest2 & 0x07) == 6) {
- rs = t2d;
- } else
- goto err;
-
- DEBUG_PRINT_BN("Squaring and shifting result (rs)", rs)
- retval = BN_copy(r, rs) != NULL;
-
-err:
- BN_CTX_end(ctx);
- return retval;
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_pss.c b/library/cpp/tvmauth/src/rw/rw_pss.c
deleted file mode 100644
index 3bf6e2b99a..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_pss.c
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * This code was taken from the OpenSSL's RSA implementation
- * and added to the RW project with some changes
- *
- * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
- * project 2005.
- *
- */
-/* ====================================================================
- * Copyright (c) 2005 The OpenSSL Project. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * 3. All advertising materials mentioning features or use of this
- * software must display the following acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
- *
- * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
- * endorse or promote products derived from this software without
- * prior written permission. For written permission, please contact
- * licensing@OpenSSL.org.
- *
- * 5. Products derived from this software may not be called "OpenSSL"
- * nor may "OpenSSL" appear in their names without prior written
- * permission of the OpenSSL Project.
- *
- * 6. Redistributions of any form whatsoever must retain the following
- * acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
- *
- * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
- * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
- * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
- * OF THE POSSIBILITY OF SUCH DAMAGE.
- * ====================================================================
- *
- * This product includes cryptographic software written by Eric Young
- * (eay@cryptsoft.com). This product includes software written by Tim
- * Hudson (tjh@cryptsoft.com).
- *
- */
-
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/bn.h>
-#include <contrib/libs/openssl/include/openssl/evp.h>
-#include <contrib/libs/openssl/include/openssl/rand.h>
-#include <contrib/libs/openssl/include/openssl/sha.h>
-
-#include <stdio.h>
-#include <string.h>
-
-static const unsigned char zeroes[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
-
-static int PkcS1MgF1(unsigned char *mask, const int len, const unsigned char *seed, const int seedlen, const EVP_MD *dgst) {
- int i, outlen = 0;
- unsigned char cnt[4];
- EVP_MD_CTX* c = EVP_MD_CTX_create();
- unsigned char md[EVP_MAX_MD_SIZE];
- int mdlen;
- int rv = -1;
-
- if (!c) {
- return rv;
- }
-
- mdlen = EVP_MD_size(dgst);
-
- if (mdlen < 0 || seedlen < 0)
- goto err;
-
- for (i = 0; outlen < len; i++) {
- cnt[0] = (unsigned char)((i >> 24) & 255);
- cnt[1] = (unsigned char)((i >> 16) & 255);
- cnt[2] = (unsigned char)((i >> 8)) & 255;
- cnt[3] = (unsigned char)(i & 255);
-
- if (!EVP_DigestInit_ex(c,dgst, NULL) || !EVP_DigestUpdate(c, seed, seedlen) || !EVP_DigestUpdate(c, cnt, 4))
- goto err;
-
- if (outlen + mdlen <= len) {
- if (!EVP_DigestFinal_ex(c, mask + outlen, NULL))
- goto err;
- outlen += mdlen;
- } else {
- if (!EVP_DigestFinal_ex(c, md, NULL))
- goto err;
- memcpy(mask + outlen, md, len - outlen);
- outlen = len;
- }
- }
- rv = 0;
-
-err:
- EVP_MD_CTX_destroy(c);
- return rv;
-}
-
-int RwVerifyPssr(const TRwKey *rw, const unsigned char *mHash, const EVP_MD *Hash, const unsigned char *EM, int sLen) {
- int i = 0, ret = 0, hLen = 0, maskedDBLen = 0, MSBits = 0, emLen = 0;
- const unsigned char *H = NULL;
- unsigned char *DB = NULL;
- EVP_MD_CTX* ctx = NULL;
- unsigned char H_[EVP_MAX_MD_SIZE];
- const EVP_MD *mgf1Hash = Hash;
-
- ctx = EVP_MD_CTX_create();
- if (!ctx) {
- return ret;
- }
- hLen = EVP_MD_size(Hash);
-
- if (hLen < 0)
- goto err;
- /*
- * Negative sLen has special meanings:
- * -1 sLen == hLen
- * -2 salt length is autorecovered from signature
- * -N reserved
- */
- if (sLen == -1)
- sLen = hLen;
- else if (sLen < -2)
- goto err;
-
- {
- int bits = BN_num_bits(rw->N);
- if (bits <= 0)
- goto err;
-
- MSBits = (bits - 1) & 0x7;
- }
- emLen = RwModSize(rw);
-
- if (EM[0] & (0xFF << MSBits)) {
- goto err;
- }
-
- if (MSBits == 0) {
- EM++;
- emLen--;
- }
-
- if (emLen < (hLen + sLen + 2)) /* sLen can be small negative */
- goto err;
-
- if (emLen < 1)
- goto err;
-
- if (EM[emLen - 1] != 0xbc)
- goto err;
-
- maskedDBLen = emLen - hLen - 1;
- if (maskedDBLen <= 0)
- goto err;
-
- H = EM + maskedDBLen;
- DB = malloc(maskedDBLen);
-
- if (!DB)
- goto err;
-
- if (PkcS1MgF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0)
- goto err;
-
- for (i = 0; i < maskedDBLen; i++)
- DB[i] ^= EM[i];
-
- if (MSBits)
- DB[0] &= 0xFF >> (8 - MSBits);
-
- for (i = 0; DB[i] == 0 && i < (maskedDBLen-1); i++) ;
-
- if (DB[i++] != 0x1)
- goto err;
-
- if (sLen >= 0 && (maskedDBLen - i) != sLen)
- goto err;
-
- if (!EVP_DigestInit_ex(ctx, Hash, NULL) || !EVP_DigestUpdate(ctx, zeroes, sizeof zeroes) || !EVP_DigestUpdate(ctx, mHash, hLen))
- goto err;
-
- if (maskedDBLen - i) {
- if (!EVP_DigestUpdate(ctx, DB + i, maskedDBLen - i))
- goto err;
- }
-
- if (!EVP_DigestFinal_ex(ctx, H_, NULL))
- goto err;
-
- ret = memcmp(H, H_, hLen) ? 0 : 1;
-
-err:
- if (DB)
- free(DB);
-
- EVP_MD_CTX_destroy(ctx);
-
- return ret;
-}
-
-/*
- rw - public key
- EM - buffer to write padding value
- mHash - hash value
- Hash - EVP_MD() that will be used to pad
- sLen - random salt len (usually == hashLen)
- */
-int RwPaddingAddPssr(const TRwKey *rw, unsigned char *EM, const unsigned char *mHash, const EVP_MD *Hash, int sLen) {
- int i = 0, ret = 0, hLen = 0, maskedDBLen = 0, MSBits = 0, emLen = 0;
- unsigned char *H = NULL, *salt = NULL, *p = NULL;
- const EVP_MD *mgf1Hash = Hash;
- EVP_MD_CTX* ctx = EVP_MD_CTX_create();
- if (!ctx) {
- return ret;
- }
-
- hLen = EVP_MD_size(Hash);
- if (hLen < 0)
- goto err;
- /*
- * Negative sLen has special meanings:
- * -1 sLen == hLen
- * -2 salt length is maximized
- * -N reserved
- */
- if (sLen == -1)
- sLen = hLen;
- else if (sLen < -2)
- goto err;
-
- {
- int bits = BN_num_bits(rw->N);
- if (bits <= 0)
- goto err;
- MSBits = (bits - 1) & 0x7;
- }
- emLen = RwModSize(rw);
- if (emLen <= 0)
- goto err;
-
- if (MSBits == 0) {
- *EM++ = 0;
- emLen--;
- fprintf(stderr, "MSBits == 0\n");
- }
-
- if (sLen == -2) {
- sLen = emLen - hLen - 2;
- }
- else if (emLen < (hLen + sLen + 2))
- goto err;
-
- if (sLen > 0) {
- salt = malloc(sLen);
- if (!salt) goto err;
- if (RAND_bytes(salt, sLen) <= 0)
- goto err;
- }
-
- maskedDBLen = emLen - hLen - 1;
- if (maskedDBLen < 0)
- goto err;
- H = EM + maskedDBLen;
-
- if (!EVP_DigestInit_ex(ctx, Hash, NULL) || !EVP_DigestUpdate(ctx, zeroes, sizeof zeroes) || !EVP_DigestUpdate(ctx, mHash, hLen))
- goto err;
-
- if (sLen && !EVP_DigestUpdate(ctx, salt, sLen))
- goto err;
-
- if (!EVP_DigestFinal_ex(ctx, H, NULL))
- goto err;
-
- /* Generate dbMask in place then perform XOR on it */
- if (PkcS1MgF1(EM, maskedDBLen, H, hLen, mgf1Hash))
- goto err;
-
- p = EM;
-
- /* Initial PS XORs with all zeroes which is a NOP so just update
- * pointer. Note from a test above this value is guaranteed to
- * be non-negative.
- */
- p += emLen - sLen - hLen - 2;
- *p++ ^= 0x1;
-
- if (sLen > 0) {
- for (i = 0; i < sLen; i++)
- *p++ ^= salt[i];
- }
-
- if (MSBits)
- EM[0] &= 0xFF >> (8 - MSBits);
-
- /* H is already in place so just set final 0xbc */
- EM[emLen - 1] = 0xbc;
-
- ret = 1;
-
-err:
- EVP_MD_CTX_destroy(ctx);
-
- if (salt)
- free(salt);
-
- return ret;
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_pss_sign.c b/library/cpp/tvmauth/src/rw/rw_pss_sign.c
deleted file mode 100644
index 59897f1cf5..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_pss_sign.c
+++ /dev/null
@@ -1,211 +0,0 @@
-#include "rw.h"
-
-#include <contrib/libs/openssl/include/openssl/evp.h>
-
-//#define DBG_FUZZING
-
-int RwApply(const int flen, const unsigned char* from, unsigned char* to, const TRwKey* rw) {
- int i, j, num, k, r = -1;
- BN_CTX* ctx = NULL;
- BIGNUM *f = NULL, *ret = NULL;
-
- if ((ctx = BN_CTX_secure_new()) == NULL)
- goto err;
- BN_CTX_start(ctx);
-
- f = BN_CTX_get(ctx);
- ret = BN_CTX_get(ctx);
-
- num = BN_num_bytes(rw->N);
-
- if (num <= 0)
- goto err;
-
- if (!f || !ret)
- goto err;
-
- if (BN_bin2bn(from, flen, f) == NULL)
- goto err;
- if (BN_ucmp(f, rw->N) >= 0)
- goto err;
-
- if (!rw->Meth->RwApply(ret, f, ctx, rw))
- goto err;
-
- j = BN_num_bytes(ret);
- if (num < j || j < 0)
- goto err;
-
- i = BN_bn2bin(ret, to + num - j);
- if (i < 0 || i > num)
- goto err;
-
- for (k = 0; k < (num - i); k++)
- to[k] = 0;
- r = num;
-
-err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
- return r;
-}
-
-int RwPssrSignHash(const unsigned char* from, unsigned char* to, TRwKey* rw, const EVP_MD* md) {
- unsigned char* padding = NULL;
- int result = 0;
-
- if (from == NULL || to == NULL || rw == NULL || md == NULL)
- return 0;
-
- int digest_size = EVP_MD_size(md);
- int sig_size = RwModSize(rw);
-
- if (digest_size <= 0 || sig_size <= 0)
- return 0;
-
- int tries = 50;
- do {
- if (padding != NULL) {
- free(padding);
-#ifdef DBG_FUZZING
- fprintf(stderr, "Padding regenerating required\n");
-#endif
- }
-
- padding = malloc(sig_size);
- if (padding == NULL)
- return 0;
-
- if (!RwPaddingAddPssr(rw, padding, from, md, digest_size))
- goto err;
- } while (padding[0] == 0x00 && tries-- > 0);
-
- result = RwNoPaddingSign(sig_size, padding, to, rw);
-
-err:
- if (padding != NULL)
- free(padding);
-
- return result;
-}
-
-int RwPssrSignMsg(const int msgLen, const unsigned char* msg, unsigned char* to, TRwKey* rw, const EVP_MD* md) {
- EVP_MD_CTX* mdctx = NULL;
- unsigned char* digest = NULL;
- unsigned int digestLen;
- int result = 0;
-
- if (msg == NULL || to == NULL || rw == NULL || md == NULL)
- goto err;
-
- if (rw->P == NULL || rw->Q == NULL)
- goto err;
-
- if ((mdctx = EVP_MD_CTX_create()) == NULL)
- goto err;
-
- if (1 != EVP_DigestInit_ex(mdctx, md, NULL))
- goto err;
-
- if (1 != EVP_DigestUpdate(mdctx, msg, msgLen))
- goto err;
-
- if ((digest = (unsigned char*)malloc(EVP_MD_size(md))) == NULL)
- goto err;
-
- if (1 != EVP_DigestFinal_ex(mdctx, digest, &digestLen))
- goto err;
-
- result = RwPssrSignHash(digest, to, rw, md);
-
-err:
- if (mdctx != NULL)
- EVP_MD_CTX_destroy(mdctx);
- if (digest != NULL)
- free(digest);
-
- return result;
-}
-
-int RwPssrVerifyHash(const unsigned char* from, const unsigned char* sig, const int sig_len, const TRwKey* rw, const EVP_MD* md) {
- unsigned char* buffer = NULL;
- int buffer_len;
- int salt_size;
- int result = 0;
-
- if (from == NULL || sig == NULL || rw == NULL || md == NULL)
- return 0;
-
- if (rw->N == NULL || rw->Meth == NULL)
- return 0;
-
- salt_size = EVP_MD_size(md);
- if (salt_size <= 0)
- return 0;
-
- buffer_len = RwModSize(rw);
- if (buffer_len <= 0)
- return 0;
-
- buffer = (unsigned char*)malloc(buffer_len);
- if (buffer == NULL)
- return 0;
-
- if (RwApply(sig_len, sig, buffer, rw) <= 0)
- goto err;
-
- if (RwVerifyPssr(rw, from, md, buffer, salt_size) <= 0)
- goto err;
-
- result = 1;
-
-err:
- if (buffer != NULL)
- free(buffer);
-
- return result;
-}
-
-int RwPssrVerifyMsg(const int msgLen, const unsigned char* msg, const unsigned char* sig, const int sig_len, const TRwKey* rw, const EVP_MD* md) {
- EVP_MD_CTX* mdctx = NULL;
- unsigned char* digest = NULL;
- unsigned int digestLen = 0;
- int result = 0;
-
- if (msg == NULL || msgLen == 0 || sig == NULL || rw == NULL || md == NULL)
- goto err;
-
- if (rw->N == NULL)
- goto err;
-
- if ((mdctx = EVP_MD_CTX_create()) == NULL)
- goto err;
-
- if (1 != EVP_DigestInit_ex(mdctx, md, NULL))
- goto err;
-
- int size_to_alloc = EVP_MD_size(md);
- if (size_to_alloc <= 0)
- goto err;
-
- if ((digest = (unsigned char*)malloc(size_to_alloc)) == NULL)
- goto err;
-
- if (1 != EVP_DigestUpdate(mdctx, msg, msgLen))
- goto err;
-
- if (1 != EVP_DigestFinal_ex(mdctx, digest, &digestLen))
- goto err;
-
- result = RwPssrVerifyHash(digest, sig, sig_len, rw, md);
-
-err:
- if (mdctx != NULL)
- EVP_MD_CTX_destroy(mdctx);
- if (digest != NULL)
- free(digest);
-
- return result;
-}
diff --git a/library/cpp/tvmauth/src/rw/rw_sign.c b/library/cpp/tvmauth/src/rw/rw_sign.c
deleted file mode 100644
index e320808dd3..0000000000
--- a/library/cpp/tvmauth/src/rw/rw_sign.c
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "rw.h"
-
-TRwSignature* RwSignatureNew(void) {
- TRwSignature* sig = NULL;
- sig = malloc(sizeof(TRwSignature));
- if (!sig)
- return NULL;
- sig->S = NULL;
- return sig;
-}
-
-void RwSignatureFree(TRwSignature* sig) {
- if (sig) {
- if (sig->S)
- BN_free(sig->S);
- free(sig);
- }
-}
-
-int RwNoPaddingSign(int flen, const unsigned char* from, unsigned char* to, TRwKey* rw) {
- int i = 0, r = 0, num = -1;
- TRwSignature* sig = NULL;
-
- if (!rw || !rw->N || !rw->Meth || !rw->Meth->RwSign || !from || !to)
- goto err;
-
- if ((sig = rw->Meth->RwSign(from, flen, rw)) == NULL)
- goto err;
- num = BN_num_bytes(rw->N);
-
- r = BN_bn2bin(sig->S, to);
- if (r < 0)
- goto err;
-
- /* put zeroes to the rest of the 'to' buffer */
- for (i = r; i < num; i++) {
- to[i] = 0x00;
- }
-
-err:
- if (sig != NULL) {
- RwSignatureFree(sig);
- }
-
- return r;
-}
diff --git a/library/cpp/tvmauth/src/rw/ut/rw_ut.cpp b/library/cpp/tvmauth/src/rw/ut/rw_ut.cpp
deleted file mode 100644
index 73f1b1d769..0000000000
--- a/library/cpp/tvmauth/src/rw/ut/rw_ut.cpp
+++ /dev/null
@@ -1,200 +0,0 @@
-#include <library/cpp/tvmauth/src/rw/keys.h>
-#include <library/cpp/tvmauth/src/rw/rw.h>
-
-#include <library/cpp/string_utils/base64/base64.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <contrib/libs/openssl/include/openssl/bn.h>
-#include <contrib/libs/openssl/include/openssl/evp.h>
-
-namespace NTvmAuth {
- /*
- returns 0 in case of error
- */
- int MakeKeysRw(TRwKey** skey, TRwKey** vkey) {
- int result = 0;
-
- TRwKey* rw = RwNew();
-
- do {
- RwGenerateKey(rw, 2048);
-
- if (rw == nullptr) {
- printf("RwGenerateKey failed\n");
- break; /* failed */
- }
-
- printf("RW key bits: %d\n", BN_num_bits(rw->N));
-
- /* Set signing key */
- *skey = RwPrivateKeyDup(rw);
- if (*skey == nullptr) {
- printf("RwPrivateKeyDup failed\n");
- break;
- }
-
- /* Set verifier key */
- *vkey = RwPublicKeyDup(rw);
- if (*vkey == nullptr) {
- printf("RwPublicKeyDup failed\n");
- break;
- }
-
- result = 1;
-
- } while (0);
-
- if (rw) {
- RwFree(rw);
- rw = nullptr;
- }
-
- return result;
- }
-
- static void PrintIt(const char* label, const unsigned char* buff, size_t len) {
- if (!buff || !len)
- return;
-
- if (label)
- printf("%s: ", label);
-
- for (size_t i = 0; i < len; ++i)
- printf("%02X", buff[i]);
-
- printf("\n");
- }
-
- int TestSignVerify() {
- TRwKey *skey = nullptr, *vkey = nullptr;
- const char* msg = "Test test test test test";
- unsigned int msg_len = 0;
- int res = 0;
-
- msg_len = (unsigned int)strlen(msg);
- if (MakeKeysRw(&skey, &vkey)) {
- unsigned char* sign = new unsigned char[RwModSize(skey) + 10];
- int sign_len;
- printf("RwModSize(skey) returned %d\n", RwModSize(skey));
- memset(sign, 0x00, RwModSize(skey) + 10);
-
- printf("--- Signing call ---\n");
- if ((sign_len = RwPssrSignMsg(msg_len, (unsigned char*)msg, sign, skey, (EVP_MD*)EVP_sha256())) != 0) {
-#ifdef RW_PRINT_DEBUG
- BIGNUM* s = BN_new();
-#endif
- printf("\n");
- PrintIt("Signature", sign, RwModSize(skey));
-
-#ifdef RW_PRINT_DEBUG
- BN_bin2bn(sign, RW_mod_size(skey), s);
-
- print_bn("Signature BN", s);
-
- BN_free(s);
-#endif
-
- printf("--- Verification call ---\n");
- if (RwPssrVerifyMsg(msg_len, (unsigned char*)msg, sign, sign_len, vkey, (EVP_MD*)EVP_sha256())) {
- printf("Verification: success!\n");
- res = 1;
- } else {
- printf("Verification: failed!\n");
- printf("RwPssrVerifyMsg failed!\n");
- return 1;
- }
- } else {
- printf("RwPssrSignMsg failed!\n");
- return 1;
- }
-
- if (sign != nullptr)
- delete[] sign;
-
- } else {
- printf("MakeKeysRw failed!\n");
- return 1;
- }
-
- if (skey != nullptr) {
- RwFree(skey);
- }
- if (vkey != nullptr)
- RwFree(vkey);
-
- return res;
- }
-}
-
-using namespace NTvmAuth;
-Y_UNIT_TEST_SUITE(Rw) {
- Y_UNIT_TEST(SignVerify) {
- for (int i = 1; i < 10; ++i) {
- UNIT_ASSERT_VALUES_EQUAL(1, TestSignVerify());
- }
- }
-
- Y_UNIT_TEST(TKeysPriv) {
- NRw::TRwPrivateKey priv(Base64Decode("MIIEmwKCAQBwsRd4frsVARIVSfj_vCdfvA3Q9SsGhSybdBDhbm8L6rPqxdoSNLCdNXzDWj7Ppf0o8uWHMxC-5Lfw0I18ri68nhm9-ndixcnbn6ti1uetgkc28eiEP6Q8ILD_JmkynbUl1aKDNAa5XsK2vFSEX402uydRomsTn46kRY23hfqcIi0ohh5VxIrpclRsRZus0JFu-RJzhqTbKYV4y4dglWPGHh5BuTv9k_Oh0_Ra8Xp5Rith5vjaKZUQ5Hyh9UtBYTkNWdvXP9OpmbiLVeRLuMzBm4HEFHDwMZ1h6LSVP-wB_spJPaMLTn3Q3JIHe-wGBYRWzU51RRYDqv4O_H12w5C1AoGBALAwCQ7fdAPG1lGclL7iWFjUofwPCFwPyDjicDT_MRRu6_Ta4GjqOGO9zuOp0o_ePgvR-7nA0fbaspM4LZNrPZwmoYBCJMtKXetg68ylu2DO-RRSN2SSh1AIZSA_8UTABk69bPzNL31j4PyZWxrgZ3zP9uZvzggveuKt5ZhCMoB7AoGBAKO9oC2AZjLdh2RaEFotTL_dY6lVcm38VA6PnigB8gB_TMuSrd4xtRw5BxvHpOCnBcUAJE0dN4_DDe5mrotKYMD2_3_lcq9PaLZadrPDCSDL89wtoVxNQNAJTqFjBFXYNu4Ze63lrsqg45TF5XmVRemyBHzXw3erd0pJaeoUDaSPAoGAJhGoHx_nVw8sDoLzeRkOJ1_6-uh_wVmVr6407_LPjrrySEq-GiYu43M3-QDp8J_J9e3S1Rpm4nQX2bEf5Gx9n4wKz7Hp0cwkOqBOWhvrAu6YLpv59wslEtkx0LYcJy6yQk5mpU8l29rPO7b50NyLnfnE2za-9DyK038FKlr5VgICgYAUd7QFsAzGW7Dsi0ILRamX-6x1Kq5Nv4qB0fPFAD5AD-mZclW7xjajhyDjePScFOC4oASJo6bx-GG9zNXRaUwYHt_v_K5V6e0Wy07WeGEkGX57hbQriagaASnULGCKuwbdwy91vLXZVBxymLyvMqi9NkCPmvhu9W7pSS09QoG0kgKBgBYGASHb7oB42sozkpfcSwsalD-B4QuB-QccTgaf5iKN3X6bXA0dRwx3udx1OlH7x8F6P3c4Gj7bVlJnBbJtZ7OE1DAIRJlpS71sHXmUt2wZ3yKKRuySUOoBDKQH_iiYAMnXrZ-Zpe-sfB-TK2NcDO-Z_tzN-cEF71xVvLMIRlAPAoGAdeikZPh1O57RxnVY72asiMRZheMBhK-9uSNPyYEZv3bUnIjg4XdMYStF2yTHNu014XvkDSQTe-drv2BDs9ExKplM4xFOtDtPQQ3mMB3GoK1qVhM_9n1QEElreurMicahkalnPo6tU4Z6PFL7PTpjRnCN67lJp0J0fxNDL13YSagCgYBA9VJrMtPjzcAx5ZCIYJjrYUPqEG_ttQN2RJIHN3MVpdpLAMIgX3tnlfyLwQFVKK45D1JgFa_1HHcxTWGtdIX4nsIjPWt-cWCCCkkw9rM5_Iqcb-YLSood6IP2OK0w0XLD1STnFRy_BRwdjPbGOYmp6YrJDZAlajDkFSdRvsz9Vg=="),
- 0);
- NRw::TRwPrivateKey priv2(Base64Decode("MIIEnAKCAQEA4RATOfumLD1n6ICrW5biaAl9VldinczmkNPjpUWwc3gs8PnkCrtdnPFmpBwW3gjHdSNU1OuEg5A6K1o1xiGv9sU-jd88zQBOdK6E2zwnJnkK6bNusKE2H2CLqg3aMWCmTa9JbzSy1uO7wa-xCqqNUuCko-2lyv12HhL1ICIH951SHDa4qO1U5xZhhlUAnqWi9R4tYDeMiF41WdOjwT2fg8UkbusThmxa3yjCXjD7OyjshPtukN8Tl3UyGtV_s2CLnE3f28VAi-AVW8FtgL22xbGhuyEplXRrtF1E5oV7NSqxH1FS0SYROA8ffYQGV5tfx5WDFHiXDEP6BzoVfeBDRQKBgQDzidelKZNFMWar_yj-r_cniMkZXNaNVEQbMg1A401blGjkU1r-ufGH5mkdNx4IgEoCEYBTM834Z88fYV1lOVfdT0OqtiVoC9NkLu3xhQ1r9_r6RMaAenwsV7leH8jWMOKvhkB0KNI49oznTGDqLp0AbDbtP66xdNH4dr3rw3WFywKBgQDslDdv4sdnRKN27h2drhn4Pp_Lgw2U-6MfHiyjp6BKR8Qtlld3hdb-ZjU9F0h38DqECmFIEe35_flKfd7X21CBQs9EuKR8EdaF3OAgzA-TRWeQhyHmaV7Fas1RlNqZHm8lckaZT8dX9Ygsxn0I_vUbm9pkFivwGvQnnwNQ7Te5LwKBgCVMYOzLHW911l6EbCZE6XU2HUrTKEd1bdqWCgtxPEmDl3BZcXpnyKpqSHmlH1F7s65WBfejxDM2hjin3OnXSog_x35ql_-Azu93-79QAzbQc6Z13BuWPpQxV8iw4ijqRRhzjD2pcvXlIxgebp5-H0eDt-Md2Y8rkrzyhm8EH7mwAoGAHZKG7fxY7OiUbt3Ds7XDPwfT-XBhsp90Y-PFlHT0CUj4hbLK7vC638zGp6LpDv4HUIFMKQI9vz-_KU-72vtqEChZ6JcUj4I60LucBBmB8mis8hDkPM0r2K1ZqjKbUyPN5K5I0yn46v6xBZjPoR_eo3N7TILFfgNehPPgah2m9yYCgYAecTr0pTJopizVf-Uf1f7k8RkjK5rRqoiDZkGoHGmrco0cimtf1z4w_M0jpuPBEAlAQjAKZnm_DPnj7Cuspyr7qeh1VsStAXpshd2-MKGtfv9fSJjQD0-Fivcrw_kaxhxV8MgOhRpHHtGc6YwdRdOgDYbdp_XWLpo_Dte9eG6wuQKBgDzo0e8d8pTyvCP23825rVzvrSHBZkliGkCEu0iggDnfKOreejFhQN9JeBo8sYdQFCRBptEU6k4b5O6J3NQ1Sspiez15ddqmFMD4uhJY6VsV-JFnL9YhLqVd355xZCyU4b07mReU9-LuqK2m2chjxH_HDAgUoEvO_yzR9EDYqHbNAoGAf529Ah9HIT5aG6IGTlwQdk-M7guy63U4vj4uC7z98qgvFEsV6cr4miT6RE8Aw5yAeN5pW59rZNjBNr9i-8n8kouasho2xNMTPKP8YuSNg2PNNS5T1Ou56mgsBCY5i10TIHKNIm2RVSUgzJ97BMEOZY6jQRytFfwgYkvnFzbuA9c="),
- 0);
- NRw::TRwPrivateKey priv3(Base64Decode("MIICVAKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NAkEAg1xBDL_UkHy347HwioMscJFP-6eKeim3LoG9rd1EvOycxkoStZ4299OdyzzEXC9cjLdq401BXe-LairiMUgZawJBALn5ziBCc2ycMaYjZDon2EN55jBEe0tJdUy4mOi0ozTV9OLcBANds0nMYPjZFOY3QymzU0LcOa_An3JknI0C2ucCQGxtwTb3h7ux5Ld8jkeRYzkNoB2Y6Is5fqCYVRIJZmz0IcQFb2iW0EX92U7_BpgVuKlvSDTP9LuaxuPfmY6WXEECQBc_OcQITm2ThjTEbIdE-whvPMYIj2lpLqmXEx0WlGaavpxbgIBrtmk5jB8bIpzG6GU2amhbhzX4E-5Mk5GgW10CQBBriCGX-pIPlvx2PhFQZY4SKf908U9FNuXQN7W7qJedk5jJQlazxt76c7lnmIuF65GW7VxpqCu98W1FXEYpAy0CQG-lpihdvxaZ8SkHqNFZGnXhELT2YesLs7GehZSTwuUwx1iTpVm88PVROLYBDZqoGM316s9aZEJBALe5zEpxQTQCQQCDMszX1cQlbBCP08isuMQ2ac3S-qNd0mfRXDCRfMm4s7iuJ5MeHU3uPUVlA_MR4ULRbg1d97TGio912z4KPgjE"),
- 0);
-
- UNIT_ASSERT_EXCEPTION(NRw::TRwPrivateKey("asdzxcv", 0), yexception);
- UNIT_ASSERT_EXCEPTION(NRw::TRwPrivateKey(Base64Decode("AKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NAkEAg1xBDL_UkHy347HwioMscJFP-6eKeim3LoG9rd1EvOycxkoStZ4299OdyzzEXC9cjLdq401BXe-LairiMUgZawJBALn5ziBCc2ycMaYjZDon2EN55jBEe0tJdUy4mOi0ozTV9OLcBANds0nMYPjZFOY3QymzU0LcOa_An3JknI0C2ucCQGxtwTb3h7ux5Ld8jkeRYzkNoB2Y6Is5fqCYVRIJZmz0IcQFb2iW0EX92U7_BpgVuKlvSDTP9LuaxuPfmY6WXEECQBc_OcQITm2ThjTEbIdE-whvPMYIj2lpLqmXEx0WlGaavpxbgIBrtmk5jB8bIpzG6GU2amhbhzX4E-5Mk5GgW10CQBBriCGX-pIPlvx2PhFQZY4SKf908U9FNuXQN7W7qJedk5jJQlazxt76c7lnmIuF65GW7VxpqCu98W1FXEYpAy0CQG-lpihdvxaZ8SkHqNFZGnXhELT2YesLs7GehZSTwuUwx1iTpVm88PVROLYBDZqoGM316s9aZEJBALe5zEpxQTQCQQCDMszX1cQlbBCP08isuMQ2ac3S-qNd0mfRXDCRfMm4s7iuJ5MeHU3uPUVlA_MR4ULRbg1d97TGio912z4KP"),
- 0),
- yexception);
-
- UNIT_ASSERT(!priv.SignTicket("").empty());
- }
-
- Y_UNIT_TEST(TKeysPub) {
- NRw::TRwPublicKey pub(Base64Decode("MIIBBAKCAQBwsRd4frsVARIVSfj_vCdfvA3Q9SsGhSybdBDhbm8L6rPqxdoSNLCdNXzDWj7Ppf0o8uWHMxC-5Lfw0I18ri68nhm9-ndixcnbn6ti1uetgkc28eiEP6Q8ILD_JmkynbUl1aKDNAa5XsK2vFSEX402uydRomsTn46kRY23hfqcIi0ohh5VxIrpclRsRZus0JFu-RJzhqTbKYV4y4dglWPGHh5BuTv9k_Oh0_Ra8Xp5Rith5vjaKZUQ5Hyh9UtBYTkNWdvXP9OpmbiLVeRLuMzBm4HEFHDwMZ1h6LSVP-wB_spJPaMLTn3Q3JIHe-wGBYRWzU51RRYDqv4O_H12w5C1"));
- NRw::TRwPublicKey pub2(Base64Decode("MIIBBQKCAQEA4RATOfumLD1n6ICrW5biaAl9VldinczmkNPjpUWwc3gs8PnkCrtdnPFmpBwW3gjHdSNU1OuEg5A6K1o1xiGv9sU-jd88zQBOdK6E2zwnJnkK6bNusKE2H2CLqg3aMWCmTa9JbzSy1uO7wa-xCqqNUuCko-2lyv12HhL1ICIH951SHDa4qO1U5xZhhlUAnqWi9R4tYDeMiF41WdOjwT2fg8UkbusThmxa3yjCXjD7OyjshPtukN8Tl3UyGtV_s2CLnE3f28VAi-AVW8FtgL22xbGhuyEplXRrtF1E5oV7NSqxH1FS0SYROA8ffYQGV5tfx5WDFHiXDEP6BzoVfeBDRQ=="));
- NRw::TRwPublicKey pub3(Base64Decode("MIGDAoGAX23ZgkYAmRFEWrp9aGLebVMVbVQ4TR_pmt9iEcCSmoaUqWHRBV95M0-l4mGLvnFfMJ7qhF5FSb7QNuoM2FNKELu4ZS_Ug1idEFBYfoT7kVzletsMVK4ZDDYRiM18fL8d58clfFAoCo-_EEMowqQeBXnxa0zqsLyNGL2x1f-KDY0="));
-
- UNIT_ASSERT_EXCEPTION(NRw::TRwPublicKey("asdzxcv"), yexception);
- UNIT_ASSERT_EXCEPTION(NRw::TRwPublicKey(Base64Decode("AoGAX23ZgkYAmRFEWrp9aGLebVMVbVQ4TR_pmt9iEcCSmoaUqWHRBV95M0-l4mGLvnFfMJ7qhF5FSb7QNuoM2FNKELu4ZS_Ug1idEFBYfoT7kVzletsMVK40")), yexception);
-
- UNIT_ASSERT(!pub.CheckSign("~~~", "~~~"));
- }
-
- Y_UNIT_TEST(TKeys) {
- NRw::TRwPrivateKey priv(Base64Decode("MIIEmwKCAQBwsRd4frsVARIVSfj_vCdfvA3Q9SsGhSybdBDhbm8L6rPqxdoSNLCdNXzDWj7Ppf0o8uWHMxC-5Lfw0I18ri68nhm9-ndixcnbn6ti1uetgkc28eiEP6Q8ILD_JmkynbUl1aKDNAa5XsK2vFSEX402uydRomsTn46kRY23hfqcIi0ohh5VxIrpclRsRZus0JFu-RJzhqTbKYV4y4dglWPGHh5BuTv9k_Oh0_Ra8Xp5Rith5vjaKZUQ5Hyh9UtBYTkNWdvXP9OpmbiLVeRLuMzBm4HEFHDwMZ1h6LSVP-wB_spJPaMLTn3Q3JIHe-wGBYRWzU51RRYDqv4O_H12w5C1AoGBALAwCQ7fdAPG1lGclL7iWFjUofwPCFwPyDjicDT_MRRu6_Ta4GjqOGO9zuOp0o_ePgvR-7nA0fbaspM4LZNrPZwmoYBCJMtKXetg68ylu2DO-RRSN2SSh1AIZSA_8UTABk69bPzNL31j4PyZWxrgZ3zP9uZvzggveuKt5ZhCMoB7AoGBAKO9oC2AZjLdh2RaEFotTL_dY6lVcm38VA6PnigB8gB_TMuSrd4xtRw5BxvHpOCnBcUAJE0dN4_DDe5mrotKYMD2_3_lcq9PaLZadrPDCSDL89wtoVxNQNAJTqFjBFXYNu4Ze63lrsqg45TF5XmVRemyBHzXw3erd0pJaeoUDaSPAoGAJhGoHx_nVw8sDoLzeRkOJ1_6-uh_wVmVr6407_LPjrrySEq-GiYu43M3-QDp8J_J9e3S1Rpm4nQX2bEf5Gx9n4wKz7Hp0cwkOqBOWhvrAu6YLpv59wslEtkx0LYcJy6yQk5mpU8l29rPO7b50NyLnfnE2za-9DyK038FKlr5VgICgYAUd7QFsAzGW7Dsi0ILRamX-6x1Kq5Nv4qB0fPFAD5AD-mZclW7xjajhyDjePScFOC4oASJo6bx-GG9zNXRaUwYHt_v_K5V6e0Wy07WeGEkGX57hbQriagaASnULGCKuwbdwy91vLXZVBxymLyvMqi9NkCPmvhu9W7pSS09QoG0kgKBgBYGASHb7oB42sozkpfcSwsalD-B4QuB-QccTgaf5iKN3X6bXA0dRwx3udx1OlH7x8F6P3c4Gj7bVlJnBbJtZ7OE1DAIRJlpS71sHXmUt2wZ3yKKRuySUOoBDKQH_iiYAMnXrZ-Zpe-sfB-TK2NcDO-Z_tzN-cEF71xVvLMIRlAPAoGAdeikZPh1O57RxnVY72asiMRZheMBhK-9uSNPyYEZv3bUnIjg4XdMYStF2yTHNu014XvkDSQTe-drv2BDs9ExKplM4xFOtDtPQQ3mMB3GoK1qVhM_9n1QEElreurMicahkalnPo6tU4Z6PFL7PTpjRnCN67lJp0J0fxNDL13YSagCgYBA9VJrMtPjzcAx5ZCIYJjrYUPqEG_ttQN2RJIHN3MVpdpLAMIgX3tnlfyLwQFVKK45D1JgFa_1HHcxTWGtdIX4nsIjPWt-cWCCCkkw9rM5_Iqcb-YLSood6IP2OK0w0XLD1STnFRy_BRwdjPbGOYmp6YrJDZAlajDkFSdRvsz9Vg=="),
- 0);
- NRw::TRwPublicKey pub(Base64Decode("MIIBBAKCAQBwsRd4frsVARIVSfj_vCdfvA3Q9SsGhSybdBDhbm8L6rPqxdoSNLCdNXzDWj7Ppf0o8uWHMxC-5Lfw0I18ri68nhm9-ndixcnbn6ti1uetgkc28eiEP6Q8ILD_JmkynbUl1aKDNAa5XsK2vFSEX402uydRomsTn46kRY23hfqcIi0ohh5VxIrpclRsRZus0JFu-RJzhqTbKYV4y4dglWPGHh5BuTv9k_Oh0_Ra8Xp5Rith5vjaKZUQ5Hyh9UtBYTkNWdvXP9OpmbiLVeRLuMzBm4HEFHDwMZ1h6LSVP-wB_spJPaMLTn3Q3JIHe-wGBYRWzU51RRYDqv4O_H12w5C1"));
-
- const TString data = "my magic data";
-
- UNIT_ASSERT(pub.CheckSign(data, priv.SignTicket(data)));
- UNIT_ASSERT(!pub.CheckSign("~~~~" + data, priv.SignTicket(data)));
- UNIT_ASSERT(!pub.CheckSign(data, "~~~~" + priv.SignTicket(data)));
-
- UNIT_ASSERT(pub.CheckSign(data,
- Base64Decode("EC5hZunmK3hOJZeov_XlNIXcwj5EsgX94lMd-tQJTNUO4NR6bCO7qQkKjEeFJmI2QFYXGY-iSf9WeMJ_brECAMyYAix-L8sZqcMPXD945QgkPsNQKyC0DX9FkgfSh6ZKkA-UvFSHrkn3QbeE9omk3-yXpqR-M8DlVqmp3mwdYlYRq0NdfTaD3AMXVA4aZTbW3OmhJoLJ8AxJ3w1oG5q_lk8dpW9vvqfIzsfPABme6sY5XyPmsjYaRDf9z4ZJgR-wTkG06_N_YzIklS5T2s_4FUKLz5gLMhsnVlNUpgZyRN9sXTAn9-zMJnCwAC8WRgykWnljPGDDJCjk-Xwsg7AOLQ==")));
- UNIT_ASSERT(pub.CheckSign(data,
- Base64Decode("JbHSn1QEQeOEvzyt-LpawbQv4vPEEE05bWhjB2-MkoV-tyq9FykSqGqhP3ZFc1_FPrqguwEYrHibI2l5w3q8wnI1fcyRUoNuJxmBSzf2f_Uzn9ZoUSc7D9pTGSvK_hhZoL4YMc_VfbdEdnDuvHZNlZyaDPH9EbmUqyXjnXTEwRoK0fAU1rhlHvSZvnp0ctVBWSkaQsaU8dJTKDBtIQVP1D5Py2pKB2NBF_Ytz2thWt7iLjbTyjtis6DC-JKwjFBqv6nQf42sKalHQqWFuIvBCIfNUswEw4_sGfwWVSBBmFplf7FmD7sN8znUahYUPGCe1uFNly6WwpPJsm8VtiU80g==")));
- UNIT_ASSERT(pub.CheckSign(data,
- Base64Decode("FeMZtDP-yuoNqK2HYw3JxTV9v7p8IoQEuRMtuHddafh4bq1ZOeEqg7g7Su6M3iq_kN9DZ_fVhuhuVcbZmNYPIvJ8oL5DE80KI3d1Qbs9mS8_X4Oq2TJpZgNfFG-z_LPRZSNRP9Q8sQhlAoSZHOSZkBFcYj1EuqEp6nSSSbX8Ji4Se-TfhIh3YFQkr-Ivk_3NmSXhDXUaW7CHo2rVm58QJ2cgSEuxzBH-Q8E8tGDCEmk4p3_iot9XY8RRN-_j0yi15etmXCUIKFbpDogtHdT8CyAEVHMYvsLqkLux9pzy3RdvNQmoPjol3wIm-H0wMtF_pMw4G2QLNev6he6xWeckxw==")));
- }
-
- Y_UNIT_TEST(Keygen) {
- for (size_t idx = 0; idx < 100; ++idx) {
- NRw::TKeyPair pair = NRw::GenKeyPair(1024);
- NRw::TRwPrivateKey priv(pair.Private, 0);
- NRw::TRwPublicKey pub(pair.Public);
-
- const TString data = "my magic data";
- TStringStream s;
- s << "data='" << data << "'.";
- s << "private='" << Base64Encode(pair.Private) << "'.";
- s << "public='" << Base64Encode(pair.Public) << "'.";
- TString sign;
- UNIT_ASSERT_NO_EXCEPTION_C(sign = priv.SignTicket(data), s.Str());
- s << "sign='" << Base64Encode(sign) << "'.";
- UNIT_ASSERT_C(pub.CheckSign(data, sign), s.Str());
- }
- }
-}
diff --git a/library/cpp/tvmauth/src/rw/ut_large/gen/main.cpp b/library/cpp/tvmauth/src/rw/ut_large/gen/main.cpp
deleted file mode 100644
index 31a599c996..0000000000
--- a/library/cpp/tvmauth/src/rw/ut_large/gen/main.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <library/cpp/tvmauth/src/rw/keys.h>
-
-#include <library/cpp/string_utils/base64/base64.h>
-
-#include <util/generic/yexception.h>
-
-using namespace NTvmAuth;
-
-const TString DATA = "my magic data";
-
-int main(int, char**) {
- const NRw::TKeyPair pair = NRw::GenKeyPair(1024);
- const NRw::TRwPrivateKey priv(pair.Private, 0);
- const NRw::TRwPublicKey pub(pair.Public);
-
- Cout << "data='" << DATA << "'."
- << "private='" << Base64Encode(pair.Private) << "'."
- << "public='" << Base64Encode(pair.Public) << "'.";
-
- TString sign;
- try {
- sign = priv.SignTicket(DATA);
- Cout << "sign='" << Base64Encode(sign) << "'.";
- Y_ENSURE(pub.CheckSign(DATA, sign));
- } catch (const std::exception& e) {
- Cout << "what='" << e.what() << "'" << Endl;
- return 1;
- }
- Cout << Endl;
-
- return 0;
-}
diff --git a/library/cpp/tvmauth/src/rw/ut_large/test.py b/library/cpp/tvmauth/src/rw/ut_large/test.py
deleted file mode 100644
index 0cf95d9848..0000000000
--- a/library/cpp/tvmauth/src/rw/ut_large/test.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from __future__ import print_function
-
-import os
-import subprocess
-import sys
-
-import yatest.common as yc
-
-
-def test_fuzzing():
- errfile = './errfile'
- outfile = './outfile'
- env = os.environ.copy()
-
- for number in range(25000):
- with open(errfile, 'w') as fe:
- with open(outfile, 'w') as fo:
- p = subprocess.Popen(
- [
- yc.build_path('library/cpp/tvmauth/src/rw/ut_large/gen/gen'),
- ],
- env=env,
- stdout=fo,
- stderr=fe,
- )
- code = p.wait()
-
- with open(errfile) as fe:
- all = fe.read()
- if all != '':
- with open(outfile) as fo:
- print(fo.read(), file=sys.stderr)
- assert all == ''
-
- assert code == 0
diff --git a/library/cpp/tvmauth/src/service_impl.cpp b/library/cpp/tvmauth/src/service_impl.cpp
deleted file mode 100644
index 528a244647..0000000000
--- a/library/cpp/tvmauth/src/service_impl.cpp
+++ /dev/null
@@ -1,203 +0,0 @@
-#include "service_impl.h"
-
-#include "parser.h"
-#include "utils.h"
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <util/generic/strbuf.h>
-#include <util/string/cast.h>
-#include <util/string/split.h>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "Method cannot be used in non-valid ticket";
-
- TCheckedServiceTicket::TImpl::operator bool() const {
- return (Status_ == ETicketStatus::Ok);
- }
-
- TTvmId TCheckedServiceTicket::TImpl::GetSrc() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return ProtobufTicket_.service().srcclientid();
- }
-
- const TScopes& TCheckedServiceTicket::TImpl::GetScopes() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- if (CachedScopes_.empty()) {
- for (const auto& el : ProtobufTicket_.service().scopes()) {
- CachedScopes_.push_back(el);
- }
- }
- return CachedScopes_;
- }
-
- bool TCheckedServiceTicket::TImpl::HasScope(TStringBuf scopeName) const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return std::binary_search(ProtobufTicket_.service().scopes().begin(), ProtobufTicket_.service().scopes().end(), scopeName);
- }
-
- ETicketStatus TCheckedServiceTicket::TImpl::GetStatus() const {
- return Status_;
- }
-
- time_t TCheckedServiceTicket::TImpl::GetExpirationTime() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return ProtobufTicket_.expirationtime();
- }
-
- TString TCheckedServiceTicket::TImpl::DebugInfo() const {
- if (CachedDebugInfo_) {
- return CachedDebugInfo_;
- }
-
- if (Status_ == ETicketStatus::Malformed) {
- CachedDebugInfo_ = "status=malformed;";
- return CachedDebugInfo_;
- }
-
- TString targetString = "ticket_type=";
- targetString.reserve(256);
- if (Status_ == ETicketStatus::InvalidTicketType) {
- targetString.append("not-serv;");
- CachedDebugInfo_ = targetString;
- return targetString;
- }
-
- targetString.append("serv");
- if (ProtobufTicket_.has_expirationtime())
- targetString.append(";expiration_time=").append(IntToString<10>(ProtobufTicket_.expirationtime()));
- if (ProtobufTicket_.service().has_srcclientid()) {
- targetString.append(";src=").append(IntToString<10>(ProtobufTicket_.service().srcclientid()));
- }
- if (ProtobufTicket_.service().has_dstclientid()) {
- targetString.append(";dst=").append(IntToString<10>(ProtobufTicket_.service().dstclientid()));
- }
- for (const auto& scope : ProtobufTicket_.service().scopes()) {
- targetString.append(";scope=").append(scope);
- }
- if (ProtobufTicket_.service().has_issueruid()) {
- targetString.append(";issuer_uid=").append(IntToString<10>(ProtobufTicket_.service().GetissuerUid()));
- }
- targetString.append(";");
-
- CachedDebugInfo_ = targetString;
- return targetString;
- }
-
- TMaybe<TUid> TCheckedServiceTicket::TImpl::GetIssuerUid() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return ProtobufTicket_.service().has_issueruid()
- ? ProtobufTicket_.service().GetissuerUid()
- : TMaybe<TUid>();
- }
-
- void TCheckedServiceTicket::TImpl::SetStatus(ETicketStatus status) {
- Status_ = status;
- }
-
- TCheckedServiceTicket::TImpl::TImpl(ETicketStatus status, ticket2::Ticket&& protobufTicket)
- : Status_(status)
- , ProtobufTicket_(std::move(protobufTicket))
- {
- }
-
- TServiceTicketImplPtr TCheckedServiceTicket::TImpl::CreateTicketForTests(ETicketStatus status,
- TTvmId src,
- TMaybe<TUid> issuerUid) {
- ticket2::Ticket proto;
- proto.mutable_service()->set_srcclientid(src);
- proto.mutable_service()->set_dstclientid(100500);
- if (issuerUid) {
- proto.mutable_service()->set_issueruid(*issuerUid);
- }
- return MakeHolder<TImpl>(status, std::move(proto));
- }
-
- TServiceContext::TImpl::TImpl(TStringBuf secretBase64, TTvmId selfTvmId, TStringBuf tvmKeysResponse)
- : Secret_(ParseSecret(secretBase64))
- , SelfTvmId_(selfTvmId)
- {
- ResetKeys(tvmKeysResponse);
- }
-
- TServiceContext::TImpl::TImpl(TTvmId selfTvmId, TStringBuf tvmKeysResponse)
- : SelfTvmId_(selfTvmId)
- {
- ResetKeys(tvmKeysResponse);
- }
-
- TServiceContext::TImpl::TImpl(TStringBuf secretBase64)
- : Secret_(ParseSecret(secretBase64))
- {
- }
-
- void TServiceContext::TImpl::ResetKeys(TStringBuf tvmKeysResponse) {
- tvm_keys::Keys protoKeys;
- if (!protoKeys.ParseFromString(TParserTvmKeys::ParseStrV1(tvmKeysResponse))) {
- ythrow TMalformedTvmKeysException() << "Malformed TVM keys";
- }
-
- NRw::TPublicKeys keys;
- for (int idx = 0; idx < protoKeys.tvm_size(); ++idx) {
- const tvm_keys::TvmKey& k = protoKeys.tvm(idx);
- keys.emplace(k.gen().id(),
- k.gen().body());
- }
-
- if (keys.empty()) {
- ythrow TEmptyTvmKeysException() << "Empty TVM keys";
- }
-
- Keys_ = std::move(keys);
- }
-
- TServiceTicketImplPtr TServiceContext::TImpl::Check(TStringBuf ticketBody) const {
- if (Keys_.empty()) {
- ythrow TEmptyTvmKeysException() << "Empty TVM keys";
- }
-
- TParserTickets::TRes res = TParserTickets::ParseV3(ticketBody, Keys_, TParserTickets::ServiceFlag());
- if (res.Status != ETicketStatus::Ok) {
- return MakeHolder<TCheckedServiceTicket::TImpl>(res.Status, std::move(res.Ticket));
- }
-
- const ETicketStatus status = CheckProtobufServiceTicket(res.Ticket);
- return MakeHolder<TCheckedServiceTicket::TImpl>(status, std::move(res.Ticket));
- }
-
- TString TServiceContext::TImpl::SignCgiParamsForTvm(TStringBuf ts, TStringBuf dst, TStringBuf scopes) const {
- if (Secret_.Value().empty()) {
- ythrow TMalformedTvmSecretException() << "Malformed TVM secret: it is empty";
- }
- return NUtils::SignCgiParamsForTvm(Secret_, ts, dst, scopes);
- }
-
- ETicketStatus TServiceContext::TImpl::CheckProtobufServiceTicket(const ticket2::Ticket& ticket) const {
- if (!ticket.has_service()) {
- return ETicketStatus::Malformed;
- }
- if (ticket.service().dstclientid() != SelfTvmId_) {
- return ETicketStatus::InvalidDst;
- }
- return ETicketStatus::Ok;
- }
-
- TString TServiceContext::TImpl::ParseSecret(TStringBuf secretBase64) {
- while (secretBase64 && secretBase64.back() == '\n') {
- secretBase64.Chop(1);
- }
-
- if (secretBase64.empty()) {
- ythrow TMalformedTvmSecretException() << "Malformed TVM secret: it is empty";
- }
-
- const TString secret = NUtils::Base64url2bin(secretBase64);
- if (secret.empty()) {
- ythrow TMalformedTvmSecretException() << "Malformed TVM secret: invalid base64url";
- }
-
- return secret;
- }
-
-}
diff --git a/library/cpp/tvmauth/src/service_impl.h b/library/cpp/tvmauth/src/service_impl.h
deleted file mode 100644
index 18dd4ec335..0000000000
--- a/library/cpp/tvmauth/src/service_impl.h
+++ /dev/null
@@ -1,77 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/src/protos/ticket2.pb.h>
-#include <library/cpp/tvmauth/src/protos/tvm_keys.pb.h>
-#include <library/cpp/tvmauth/src/rw/keys.h>
-
-#include <library/cpp/tvmauth/type.h>
-#include <library/cpp/tvmauth/deprecated/service_context.h>
-
-#include <library/cpp/charset/ci_string.h>
-#include <library/cpp/string_utils/secret_string/secret_string.h>
-
-#include <util/generic/maybe.h>
-
-#include <string>
-
-namespace NTvmAuth {
- using TServiceTicketImplPtr = THolder<TCheckedServiceTicket::TImpl>;
- class TCheckedServiceTicket::TImpl {
- public:
- explicit operator bool() const;
-
- TTvmId GetSrc() const;
- const TScopes& GetScopes() const;
- bool HasScope(TStringBuf scopeName) const;
- ETicketStatus GetStatus() const;
- time_t GetExpirationTime() const;
-
- TString DebugInfo() const;
- TMaybe<TUid> GetIssuerUid() const;
-
- void SetStatus(ETicketStatus status);
-
- /*!
- * Constructor for creation invalid ticket storing error status in TServiceContext
- * @param status
- * @param protobufTicket
- */
- TImpl(ETicketStatus status, ticket2::Ticket&& protobufTicket);
-
- static TServiceTicketImplPtr CreateTicketForTests(ETicketStatus status,
- TTvmId src,
- TMaybe<TUid> issuerUid);
-
- private:
- ETicketStatus Status_;
- ticket2::Ticket ProtobufTicket_;
- mutable TScopes CachedScopes_;
- mutable TString CachedDebugInfo_;
- };
-
- class TServiceContext::TImpl {
- public:
- TImpl(TStringBuf secretBase64, TTvmId selfTvmId, TStringBuf tvmKeysResponse);
- TImpl(TTvmId selfTvmId, TStringBuf tvmKeysResponse);
- TImpl(TStringBuf secretBase64);
-
- void ResetKeys(TStringBuf tvmKeysResponse);
-
- TServiceTicketImplPtr Check(TStringBuf ticketBody) const;
- TString SignCgiParamsForTvm(TStringBuf ts, TStringBuf dst, TStringBuf scopes = TStringBuf()) const;
-
- const NRw::TPublicKeys& GetKeys() const { // for tests
- return Keys_;
- }
-
- private:
- ETicketStatus CheckProtobufServiceTicket(const ticket2::Ticket& ticket) const;
- static TString ParseSecret(TStringBuf secretBase64);
-
- NRw::TPublicKeys Keys_;
- const NSecretString::TSecretString Secret_;
- const TTvmId SelfTvmId_ = 0;
-
- ::google::protobuf::LogSilencer LogSilencer_;
- };
-}
diff --git a/library/cpp/tvmauth/src/service_ticket.cpp b/library/cpp/tvmauth/src/service_ticket.cpp
deleted file mode 100644
index 077049ef3a..0000000000
--- a/library/cpp/tvmauth/src/service_ticket.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-#include "service_impl.h"
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "Ticket already moved out";
-
- TCheckedServiceTicket::TCheckedServiceTicket(THolder<TImpl> impl)
- : Impl_(std::move(impl))
- {
- }
-
- TCheckedServiceTicket::TCheckedServiceTicket(TCheckedServiceTicket&& o) = default;
- TCheckedServiceTicket& TCheckedServiceTicket::operator=(TCheckedServiceTicket&& o) = default;
- TCheckedServiceTicket::~TCheckedServiceTicket() = default;
-
- TCheckedServiceTicket::operator bool() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->operator bool();
- }
-
- TTvmId TCheckedServiceTicket::GetSrc() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetSrc();
- }
-
- ETicketStatus TCheckedServiceTicket::GetStatus() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetStatus();
- }
-
- TString TCheckedServiceTicket::DebugInfo() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->DebugInfo();
- }
-
- TMaybe<TUid> TCheckedServiceTicket::GetIssuerUid() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetIssuerUid();
- }
-}
diff --git a/library/cpp/tvmauth/src/status.cpp b/library/cpp/tvmauth/src/status.cpp
deleted file mode 100644
index 1b08fc098f..0000000000
--- a/library/cpp/tvmauth/src/status.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <util/generic/yexception.h>
-
-namespace NTvmAuth {
- TStringBuf StatusToString(ETicketStatus st) {
- switch (st) {
- case ETicketStatus::Ok:
- return "OK";
- case ETicketStatus::Expired:
- return "Expired ticket";
- case ETicketStatus::InvalidBlackboxEnv:
- return "Invalid BlackBox environment";
- case ETicketStatus::InvalidDst:
- return "Invalid ticket destination";
- case ETicketStatus::InvalidTicketType:
- return "Invalid ticket type";
- case ETicketStatus::Malformed:
- return "Malformed ticket";
- case ETicketStatus::MissingKey:
- return "Context does not have required key to check ticket: public keys are too old";
- case ETicketStatus::SignBroken:
- return "Invalid ticket signature";
- case ETicketStatus::UnsupportedVersion:
- return "Unsupported ticket version";
- case ETicketStatus::NoRoles:
- return "Subject (src or defaultUid) does not have any roles in IDM";
- }
-
- ythrow yexception() << "Unexpected status: " << static_cast<int>(st);
- }
-}
diff --git a/library/cpp/tvmauth/src/unittest.cpp b/library/cpp/tvmauth/src/unittest.cpp
deleted file mode 100644
index 5133d79ea9..0000000000
--- a/library/cpp/tvmauth/src/unittest.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "service_impl.h"
-#include "user_impl.h"
-
-#include <library/cpp/tvmauth/unittest.h>
-
-namespace NTvmAuth::NUnittest {
- TCheckedServiceTicket CreateServiceTicket(ETicketStatus status, TTvmId src, TMaybe<TUid> issuerUid) {
- return TCheckedServiceTicket(TCheckedServiceTicket::TImpl::CreateTicketForTests(status, src, issuerUid));
- }
-
- TCheckedUserTicket CreateUserTicket(ETicketStatus status, TUid defaultUid, const TScopes& scopes, const TUids& uids, EBlackboxEnv env) {
- return TCheckedUserTicket(TCheckedUserTicket::TImpl::CreateTicketForTests(status, defaultUid, scopes, uids, env));
- }
-}
diff --git a/library/cpp/tvmauth/src/user_impl.cpp b/library/cpp/tvmauth/src/user_impl.cpp
deleted file mode 100644
index 4fda799aac..0000000000
--- a/library/cpp/tvmauth/src/user_impl.cpp
+++ /dev/null
@@ -1,241 +0,0 @@
-#include "user_impl.h"
-
-#include "parser.h"
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <util/generic/strbuf.h>
-#include <util/string/cast.h>
-#include <util/string/split.h>
-
-#include <algorithm>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "Method cannot be used in non-valid ticket";
-
- TStringBuf GetBlackboxEnvAsString(EBlackboxEnv environment) {
- switch (environment) {
- case (EBlackboxEnv::Prod):
- return TStringBuf("Prod");
- case (EBlackboxEnv::Test):
- return TStringBuf("Test");
- case (EBlackboxEnv::ProdYateam):
- return TStringBuf("ProdYateam");
- case (EBlackboxEnv::TestYateam):
- return TStringBuf("TestYateam");
- case (EBlackboxEnv::Stress):
- return TStringBuf("Stress");
- default:
- throw yexception() << "Unknown environment";
- }
- }
-
- TCheckedUserTicket::TImpl::operator bool() const {
- return (Status_ == ETicketStatus::Ok);
- }
-
- TUid TCheckedUserTicket::TImpl::GetDefaultUid() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return ProtobufTicket_.user().defaultuid();
- }
-
- time_t TCheckedUserTicket::TImpl::GetExpirationTime() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return ProtobufTicket_.expirationtime();
- }
-
- const TScopes& TCheckedUserTicket::TImpl::GetScopes() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- if (CachedScopes_.empty()) {
- for (const auto& el : ProtobufTicket_.user().scopes()) {
- CachedScopes_.push_back(el);
- }
- }
- return CachedScopes_;
- }
-
- bool TCheckedUserTicket::TImpl::HasScope(TStringBuf scopeName) const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- return std::binary_search(ProtobufTicket_.user().scopes().begin(), ProtobufTicket_.user().scopes().end(), scopeName);
- }
-
- ETicketStatus TCheckedUserTicket::TImpl::GetStatus() const {
- return Status_;
- }
-
- const TUids& TCheckedUserTicket::TImpl::GetUids() const {
- Y_ENSURE_EX(bool(*this), TNotAllowedException() << EX_MSG);
- if (CachedUids_.empty()) {
- for (const auto& user : ProtobufTicket_.user().users()) {
- CachedUids_.push_back(user.uid());
- }
- }
- return CachedUids_;
- }
-
- TString TCheckedUserTicket::TImpl::DebugInfo() const {
- if (CachedDebugInfo_) {
- return CachedDebugInfo_;
- }
-
- if (Status_ == ETicketStatus::Malformed) {
- CachedDebugInfo_ = "status=malformed;";
- return CachedDebugInfo_;
- }
-
- TString targetString = "ticket_type=";
- targetString.reserve(256);
- if (Status_ == ETicketStatus::InvalidTicketType) {
- targetString.append("not-user;");
- CachedDebugInfo_ = targetString;
- return targetString;
- }
-
- targetString.append("user");
- if (ProtobufTicket_.expirationtime() > 0)
- targetString.append(";expiration_time=").append(IntToString<10>(ProtobufTicket_.expirationtime()));
- for (const auto& scope : ProtobufTicket_.user().scopes()) {
- targetString.append(";scope=").append(scope);
- }
-
- if (ProtobufTicket_.user().defaultuid() > 0)
- targetString.append(";default_uid=").append(IntToString<10>(ProtobufTicket_.user().defaultuid()));
- for (const auto& user : ProtobufTicket_.user().users()) {
- targetString.append(";uid=").append(IntToString<10>(user.uid()));
- }
-
- targetString.append(";env=");
- EBlackboxEnv environment = static_cast<EBlackboxEnv>(ProtobufTicket_.user().env());
- targetString.append(GetBlackboxEnvAsString(environment));
- targetString.append(";");
-
- CachedDebugInfo_ = targetString;
- return targetString;
- }
-
- EBlackboxEnv TCheckedUserTicket::TImpl::GetEnv() const {
- return (EBlackboxEnv)ProtobufTicket_.user().env();
- }
-
- void TCheckedUserTicket::TImpl::SetStatus(ETicketStatus status) {
- Status_ = status;
- }
-
- TCheckedUserTicket::TImpl::TImpl(ETicketStatus status, ticket2::Ticket&& protobufTicket)
- : Status_(status)
- , ProtobufTicket_(std::move(protobufTicket))
- {
- }
-
- TUserTicketImplPtr TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus status,
- TUid defaultUid,
- TScopes scopes,
- TUids uids,
- EBlackboxEnv env) {
- auto prepareCont = [](auto& cont) {
- std::sort(cont.begin(), cont.end());
- cont.erase(std::unique(cont.begin(), cont.end()), cont.end());
- };
- auto erase = [](auto& cont, auto val) {
- auto it = std::find(cont.begin(), cont.end(), val);
- if (it != cont.end()) {
- cont.erase(it);
- }
- };
-
- prepareCont(scopes);
- erase(scopes, "");
-
- uids.push_back(defaultUid);
- prepareCont(uids);
- erase(uids, 0);
- Y_ENSURE(!uids.empty(), "User ticket cannot contain empty uid list");
-
- ticket2::Ticket proto;
- for (TUid uid : uids) {
- proto.mutable_user()->add_users()->set_uid(uid);
- }
- proto.mutable_user()->set_defaultuid(defaultUid);
- proto.mutable_user()->set_entrypoint(100500);
- for (TStringBuf scope : scopes) {
- proto.mutable_user()->add_scopes(TString(scope));
- }
-
- proto.mutable_user()->set_env((tvm_keys::BbEnvType)env);
-
- return MakeHolder<TImpl>(status, std::move(proto));
- }
-
- TUserContext::TImpl::TImpl(EBlackboxEnv env, TStringBuf tvmKeysResponse)
- : Env_(env)
- {
- ResetKeys(tvmKeysResponse);
- }
-
- void TUserContext::TImpl::ResetKeys(TStringBuf tvmKeysResponse) {
- tvm_keys::Keys protoKeys;
- if (!protoKeys.ParseFromString(TParserTvmKeys::ParseStrV1(tvmKeysResponse))) {
- ythrow TMalformedTvmKeysException() << "Malformed TVM keys";
- }
-
- NRw::TPublicKeys keys;
- for (int idx = 0; idx < protoKeys.bb_size(); ++idx) {
- const tvm_keys::BbKey& k = protoKeys.bb(idx);
- if (IsAllowed(k.env())) {
- keys.emplace(k.gen().id(),
- k.gen().body());
- }
- }
-
- if (keys.empty()) {
- ythrow TEmptyTvmKeysException() << "Empty TVM keys";
- }
-
- Keys_ = std::move(keys);
- }
-
- TUserTicketImplPtr TUserContext::TImpl::Check(TStringBuf ticketBody) const {
- TParserTickets::TRes res = TParserTickets::ParseV3(ticketBody, Keys_, TParserTickets::UserFlag());
- ETicketStatus status = CheckProtobufUserTicket(res.Ticket);
-
- if (res.Status != ETicketStatus::Ok && !(res.Status == ETicketStatus::MissingKey && status == ETicketStatus::InvalidBlackboxEnv)) {
- status = res.Status;
- }
- return MakeHolder<TCheckedUserTicket::TImpl>(status, std::move(res.Ticket));
- }
-
- ETicketStatus TUserContext::TImpl::CheckProtobufUserTicket(const ticket2::Ticket& ticket) const {
- if (!ticket.has_user()) {
- return ETicketStatus::Malformed;
- }
- if (!IsAllowed(ticket.user().env())) {
- return ETicketStatus::InvalidBlackboxEnv;
- }
- return ETicketStatus::Ok;
- }
-
- const NRw::TPublicKeys& TUserContext::TImpl::GetKeys() const {
- return Keys_;
- }
-
- bool TUserContext::TImpl::IsAllowed(tvm_keys::BbEnvType env) const {
- if (env == tvm_keys::Prod && (Env_ == EBlackboxEnv::Prod || Env_ == EBlackboxEnv::Stress)) {
- return true;
- }
- if (env == tvm_keys::ProdYateam && Env_ == EBlackboxEnv::ProdYateam) {
- return true;
- }
- if (env == tvm_keys::Test && Env_ == EBlackboxEnv::Test) {
- return true;
- }
- if (env == tvm_keys::TestYateam && Env_ == EBlackboxEnv::TestYateam) {
- return true;
- }
- if (env == tvm_keys::Stress && Env_ == EBlackboxEnv::Stress) {
- return true;
- }
-
- return false;
- }
-}
diff --git a/library/cpp/tvmauth/src/user_impl.h b/library/cpp/tvmauth/src/user_impl.h
deleted file mode 100644
index e3f1099b90..0000000000
--- a/library/cpp/tvmauth/src/user_impl.h
+++ /dev/null
@@ -1,72 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/src/protos/ticket2.pb.h>
-#include <library/cpp/tvmauth/src/protos/tvm_keys.pb.h>
-#include <library/cpp/tvmauth/src/rw/keys.h>
-
-#include <library/cpp/tvmauth/deprecated/user_context.h>
-
-#include <library/cpp/charset/ci_string.h>
-
-#include <unordered_map>
-
-namespace NTvmAuth {
- using TUserTicketImplPtr = THolder<TCheckedUserTicket::TImpl>;
- class TCheckedUserTicket::TImpl {
- public:
- explicit operator bool() const;
-
- TUid GetDefaultUid() const;
- time_t GetExpirationTime() const;
- const TScopes& GetScopes() const;
- bool HasScope(TStringBuf scopeName) const;
- ETicketStatus GetStatus() const;
- const TUids& GetUids() const;
-
- TString DebugInfo() const;
-
- EBlackboxEnv GetEnv() const;
-
- void SetStatus(ETicketStatus status);
-
- /*!
- * Constructor for creation invalid ticket storing error status in TServiceContext
- * @param status
- * @param protobufTicket
- */
- TImpl(ETicketStatus status, ticket2::Ticket&& protobufTicket);
-
- static TUserTicketImplPtr CreateTicketForTests(ETicketStatus status,
- TUid defaultUid,
- TScopes scopes,
- TUids uids,
- EBlackboxEnv env = EBlackboxEnv::Test);
-
- private:
- static const int MaxUserCount = 15;
-
- ETicketStatus Status_;
- ticket2::Ticket ProtobufTicket_;
- mutable TScopes CachedScopes_;
- mutable TUids CachedUids_;
- mutable TString CachedDebugInfo_;
- };
-
- class TUserContext::TImpl {
- public:
- TImpl(EBlackboxEnv env, TStringBuf tvmKeysResponse);
- void ResetKeys(TStringBuf tvmKeysResponse);
-
- TUserTicketImplPtr Check(TStringBuf ticketBody) const;
- const NRw::TPublicKeys& GetKeys() const;
-
- bool IsAllowed(tvm_keys::BbEnvType env) const;
-
- private:
- ETicketStatus CheckProtobufUserTicket(const ticket2::Ticket& ticket) const;
-
- NRw::TPublicKeys Keys_;
- EBlackboxEnv Env_;
- ::google::protobuf::LogSilencer LogSilencer_;
- };
-}
diff --git a/library/cpp/tvmauth/src/user_ticket.cpp b/library/cpp/tvmauth/src/user_ticket.cpp
deleted file mode 100644
index 3e4e0c0364..0000000000
--- a/library/cpp/tvmauth/src/user_ticket.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-#include "user_impl.h"
-
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-
-namespace NTvmAuth {
- static const char* EX_MSG = "Ticket already moved out";
-
- TCheckedUserTicket::TCheckedUserTicket(THolder<TCheckedUserTicket::TImpl> impl)
- : Impl_(std::move(impl))
- {
- }
-
- TCheckedUserTicket::TCheckedUserTicket(TCheckedUserTicket&& o) = default;
- TCheckedUserTicket::~TCheckedUserTicket() = default;
- TCheckedUserTicket& TCheckedUserTicket::operator=(TCheckedUserTicket&& o) = default;
-
- TCheckedUserTicket::operator bool() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->operator bool();
- }
-
- const TUids& TCheckedUserTicket::GetUids() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetUids();
- }
-
- TUid TCheckedUserTicket::GetDefaultUid() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetDefaultUid();
- }
-
- const TScopes& TCheckedUserTicket::GetScopes() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetScopes();
- }
-
- bool TCheckedUserTicket::HasScope(TStringBuf scopeName) const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->HasScope(scopeName);
- }
-
- ETicketStatus TCheckedUserTicket::GetStatus() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetStatus();
- }
-
- TString TCheckedUserTicket::DebugInfo() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->DebugInfo();
- }
-
- EBlackboxEnv TCheckedUserTicket::GetEnv() const {
- Y_ENSURE(Impl_, EX_MSG);
- return Impl_->GetEnv();
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/parser_ut.cpp b/library/cpp/tvmauth/src/ut/parser_ut.cpp
deleted file mode 100644
index 530f45331a..0000000000
--- a/library/cpp/tvmauth/src/ut/parser_ut.cpp
+++ /dev/null
@@ -1,143 +0,0 @@
-#include <library/cpp/tvmauth/src/parser.h>
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-Y_UNIT_TEST_SUITE(ParserTestSuite) {
- using namespace NTvmAuth;
-
- Y_UNIT_TEST(Keys) {
- UNIT_ASSERT_EXCEPTION(TParserTvmKeys::ParseStrV1("2:asds"), TMalformedTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TParserTvmKeys::ParseStrV1("3:asds"), TMalformedTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TParserTvmKeys::ParseStrV1("1:+a/sds"), TMalformedTvmKeysException);
-
- UNIT_ASSERT_VALUES_EQUAL("sdsd", NUtils::Bin2base64url(TParserTvmKeys::ParseStrV1("1:sdsd")));
- }
-
- Y_UNIT_TEST(TicketsStrV3) {
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Ok,
- NUtils::Base64url2bin("CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg"),
- NUtils::Base64url2bin("ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA"),
- "3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:"}),
- TParserTickets::ParseStrV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::UnsupportedVersion,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("2:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::InvalidTicketType,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::UserFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv::ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA:asd",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv:CgY+-*/IDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERme/*-+H_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("",
- TParserTickets::ServiceFlag()));
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Malformed,
- {},
- {},
- {}}),
- TParserTickets::ParseStrV3("'",
- TParserTickets::ServiceFlag()));
-
- // Invalid proto
- UNIT_ASSERT_EQUAL(TParserTickets::TStrRes({ETicketStatus::Ok,
- NUtils::Base64url2bin("YIDRCUkQYBgcIgdiYjpzZXNzIghiYjpzZXNzMg"),
- NUtils::Base64url2bin("ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA"),
- "3:serv:YIDRCUkQYBgcIgdiYjpzZXNzIghiYjpzZXNzMg:"}),
- TParserTickets::ParseStrV3("3:serv:YIDRCUkQYBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- TParserTickets::ServiceFlag()));
- }
-
- Y_UNIT_TEST(TicketsV3) {
- NRw::TPublicKeys pub;
-
- UNIT_ASSERT_EQUAL(ETicketStatus::Malformed,
- TParserTickets::ParseV3("3:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERme/*-+H_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
-
- // Invalid proto
- UNIT_ASSERT_EQUAL(ETicketStatus::Malformed,
- TParserTickets::ParseV3("3:serv:YIDRCUkQYBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
-
- // Expire time == 100500
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired,
- TParserTickets::ParseV3("3:serv:CBAQlJEGIhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMg:HEzPbsjULegBvgX3nqwFX0GfVhESmN1kEWyeT7U03KAR-sQnNYgm6IuN-b9-lQYQKAJSW6p8ffyucC1yDrWSWRxXVzHJUxAVW4hnbiFDtXrurnEdpMK3izKbmTY25PJ4vH3_TkRXk-_oSAE8RvIFKXlh-aw1tezbXBUpJKvyJ0w",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
-
- UNIT_ASSERT_EQUAL(ETicketStatus::MissingKey,
- TParserTickets::ParseV3("3:serv:CBAQ__________9_IhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMg:OKjKEbygehEZWH0XEeLzvf0q0aS0VvSk_CKSXGdpqxPbE4RzU70jeM-X9rXVpbYjt76VgBLlBpumJdyiclulfGPDPiL8nwJuu8AnWIR_o-QqyXbsloo2_syE6w2aYw2Yw_5_qjnipYdxGUWegHAGCj3yeMde6O2BmNZ0OCfg6qU",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
-
- pub.emplace(16, NRw::TRwPublicKey(NUtils::Base64url2bin("MIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbN")));
- UNIT_ASSERT_EQUAL(ETicketStatus::SignBroken,
- TParserTickets::ParseV3("3:serv:CBAQ__________9_IhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMa:OKjKEbygehEZWH0XEeLzvf0q0aS0VvSk_CKSXGdpqxPbE4RzU70jeM-X9rXVpbYjt76VgBLlBpumJdyiclulfGPDPiL8nwJuu8AnWIR_o-QqyXbsloo2_syE6w2aYw2Yw_5_qjnipYdxGUWegHAGCj3yeMde6O2BmNZ0OCfg6qU",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
- UNIT_ASSERT_EQUAL(ETicketStatus::SignBroken,
- TParserTickets::ParseV3("3:serv:CBAQ__________9_IhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMg:OKjKEbygehEZWH0XEeLzvf0q0aS0VvSk_CKSXGdpqxPbE4RzU70jeM-X9rXVpbYjt76VgBLlBpumJdyiclulfGPDPiL8nwJuu8AnWIR_o-QqyXbsloo2_syE6w2aYw2Yw_5_qjnipYdxGUWegHAGCj3yeMde6O2BmNZ0OCfg6qa",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
- UNIT_ASSERT_EQUAL(ETicketStatus::SignBroken,
- TParserTickets::ParseV3("3:serv:CBAQ__________9_IhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMg:EbygehEZWH0XEeLzvf0q0aS0VvSk_CKSXGdpqxPbE4RzU70jeM-X9rXVpbYjt76VgBLlBpumJdyiclulfGPDPiL8nwJuu8AnWIR_o-QqyXbsloo2_syE6w2aYw2Yw_5_qjnipYdxGUWegHAGCj3yeMde6O2BmNZ0OCfg6qU",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
-
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok,
- TParserTickets::ParseV3("3:serv:CBAQ__________9_IhcIDBAcGgdiYjpzZXNzGghiYjpzZXNzMg:OKjKEbygehEZWH0XEeLzvf0q0aS0VvSk_CKSXGdpqxPbE4RzU70jeM-X9rXVpbYjt76VgBLlBpumJdyiclulfGPDPiL8nwJuu8AnWIR_o-QqyXbsloo2_syE6w2aYw2Yw_5_qjnipYdxGUWegHAGCj3yeMde6O2BmNZ0OCfg6qU",
- pub,
- TParserTickets::ServiceFlag())
- .Status);
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/public_ut.cpp b/library/cpp/tvmauth/src/ut/public_ut.cpp
deleted file mode 100644
index 74a483d57b..0000000000
--- a/library/cpp/tvmauth/src/ut/public_ut.cpp
+++ /dev/null
@@ -1,290 +0,0 @@
-// DO_NOT_STYLE
-#include <library/cpp/tvmauth/src/service_impl.h>
-#include <library/cpp/tvmauth/src/user_impl.h>
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/ticket_status.h>
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(CommonPublicInterfaceTestSuite){
- Y_UNIT_TEST(StatusTest){
- UNIT_ASSERT_VALUES_EQUAL("OK",
- StatusToString(ETicketStatus::Ok));
- UNIT_ASSERT_VALUES_EQUAL("Expired ticket",
- StatusToString(ETicketStatus::Expired));
- UNIT_ASSERT_VALUES_EQUAL("Invalid BlackBox environment",
- StatusToString(ETicketStatus::InvalidBlackboxEnv));
- UNIT_ASSERT_VALUES_EQUAL("Invalid ticket destination",
- StatusToString(ETicketStatus::InvalidDst));
- UNIT_ASSERT_VALUES_EQUAL("Invalid ticket type",
- StatusToString(ETicketStatus::InvalidTicketType));
- UNIT_ASSERT_VALUES_EQUAL("Malformed ticket",
- StatusToString(ETicketStatus::Malformed));
- UNIT_ASSERT_VALUES_EQUAL("Invalid ticket signature",
- StatusToString(ETicketStatus::SignBroken));
- UNIT_ASSERT_VALUES_EQUAL("Context does not have required key to check ticket: public keys are too old",
- StatusToString(ETicketStatus::MissingKey));
- UNIT_ASSERT_VALUES_EQUAL("Unsupported ticket version",
- StatusToString(ETicketStatus::UnsupportedVersion));
- }
-}
-
-Y_UNIT_TEST_SUITE(PublicInterfaceServiceTestSuite) {
- static const TString EMPTY_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAE";
- static const TString EXPIRED_SERVICE_TICKET = "3:serv:CBAQACIZCOUBEBwaCGJiOnNlc3MxGghiYjpzZXNzMg:IwfMNJYEqStY_SixwqJnyHOMCPR7-3HHk4uylB2oVRkthtezq-OOA7QizDvx7VABLs_iTlXuD1r5IjufNei_EiV145eaa3HIg4xCdJXCojMexf2UYJz8mF2b0YzFAy6_KWagU7xo13CyKAqzJuQf5MJcSUf0ecY9hVh36cJ51aw";
- static const TString MALFORMED_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAEEpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQYSlQEKkgEIEBAAGocBMIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbNIJ";
- static const TString MALFORMED_TVM_SECRET = "adcvxcv./-+";
- static const TTvmId NOT_OUR_ID = 27;
- static const TTvmId OUR_ID = 28;
- static const TString SECRET = "GRMJrKnj4fOVnvOqe-WyD1";
- static const TString SERVICE_TICKET_PROTOBUF = "CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My";
- static const TTvmId SRC_ID = 229;
- static const TString UNSUPPORTED_VERSION_SERVICE_TICKET = "2:serv:CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My:WUPx1cTf05fjD1exB35T5j2DCHWH1YaLJon_a4rN-D7JfXHK1Ai4wM4uSfboHD9xmGQH7extqtlEk1tCTCGm5qbRVloJwWzCZBXo3zKX6i1oBYP_89WcjCNPVe1e8jwGdLsnu6PpxL5cn0xCksiStILH5UmDR6xfkJdnmMG94o8";
- static const TString VALID_SERVICE_TICKET_1 = "3:serv:CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My:WUPx1cTf05fjD1exB35T5j2DCHWH1YaLJon_a4rN-D7JfXHK1Ai4wM4uSfboHD9xmGQH7extqtlEk1tCTCGm5qbRVloJwWzCZBXo3zKX6i1oBYP_89WcjCNPVe1e8jwGdLsnu6PpxL5cn0xCksiStILH5UmDR6xfkJdnmMG94o8";
- static const TString VALID_SERVICE_TICKET_2 = "3:serv:CBAQ__________9_IskICOUBEBwaCGJiOnNlc3MxGgliYjpzZXNzMTAaCmJiOnNlc3MxMDAaCWJiOnNlc3MxMRoJYmI6c2VzczEyGgliYjpzZXNzMTMaCWJiOnNlc3MxNBoJYmI6c2VzczE1GgliYjpzZXNzMTYaCWJiOnNlc3MxNxoJYmI6c2VzczE4GgliYjpzZXNzMTkaCGJiOnNlc3MyGgliYjpzZXNzMjAaCWJiOnNlc3MyMRoJYmI6c2VzczIyGgliYjpzZXNzMjMaCWJiOnNlc3MyNBoJYmI6c2VzczI1GgliYjpzZXNzMjYaCWJiOnNlc3MyNxoJYmI6c2VzczI4GgliYjpzZXNzMjkaCGJiOnNlc3MzGgliYjpzZXNzMzAaCWJiOnNlc3MzMRoJYmI6c2VzczMyGgliYjpzZXNzMzMaCWJiOnNlc3MzNBoJYmI6c2VzczM1GgliYjpzZXNzMzYaCWJiOnNlc3MzNxoJYmI6c2VzczM4GgliYjpzZXNzMzkaCGJiOnNlc3M0GgliYjpzZXNzNDAaCWJiOnNlc3M0MRoJYmI6c2VzczQyGgliYjpzZXNzNDMaCWJiOnNlc3M0NBoJYmI6c2VzczQ1GgliYjpzZXNzNDYaCWJiOnNlc3M0NxoJYmI6c2VzczQ4GgliYjpzZXNzNDkaCGJiOnNlc3M1GgliYjpzZXNzNTAaCWJiOnNlc3M1MRoJYmI6c2VzczUyGgliYjpzZXNzNTMaCWJiOnNlc3M1NBoJYmI6c2VzczU1GgliYjpzZXNzNTYaCWJiOnNlc3M1NxoJYmI6c2VzczU4GgliYjpzZXNzNTkaCGJiOnNlc3M2GgliYjpzZXNzNjAaCWJiOnNlc3M2MRoJYmI6c2VzczYyGgliYjpzZXNzNjMaCWJiOnNlc3M2NBoJYmI6c2VzczY1GgliYjpzZXNzNjYaCWJiOnNlc3M2NxoJYmI6c2VzczY4GgliYjpzZXNzNjkaCGJiOnNlc3M3GgliYjpzZXNzNzAaCWJiOnNlc3M3MRoJYmI6c2VzczcyGgliYjpzZXNzNzMaCWJiOnNlc3M3NBoJYmI6c2Vzczc1GgliYjpzZXNzNzYaCWJiOnNlc3M3NxoJYmI6c2Vzczc4GgliYjpzZXNzNzkaCGJiOnNlc3M4GgliYjpzZXNzODAaCWJiOnNlc3M4MRoJYmI6c2VzczgyGgliYjpzZXNzODMaCWJiOnNlc3M4NBoJYmI6c2Vzczg1GgliYjpzZXNzODYaCWJiOnNlc3M4NxoJYmI6c2Vzczg4GgliYjpzZXNzODkaCGJiOnNlc3M5GgliYjpzZXNzOTAaCWJiOnNlc3M5MRoJYmI6c2VzczkyGgliYjpzZXNzOTMaCWJiOnNlc3M5NBoJYmI6c2Vzczk1GgliYjpzZXNzOTYaCWJiOnNlc3M5NxoJYmI6c2Vzczk4GgliYjpzZXNzOTk:JYmABAVLM6y7_T4n1pRcwBfwDfzMV4JJ3cpbEG617zdGgKRZwL7MalsYn5bq1F2ibujMrsF9nzZf8l4s_e-Ivjkz_xu4KMzSp-pUh9V7XIF_smj0WHYpv6gOvWNuK8uIvlZTTKwtQX0qZOL9m-MEeZiHoQPKZGCfJ_qxMUp-J8I";
- static const TString VALID_SERVICE_TICKET_3 = "3:serv:CBAQ__________9_IgUI5QEQHA:Sd6tmA1CNy2Nf7XevC3x7zr2DrGNRmcl-TxUsDtDW2xI3YXyCxBltWeg0-KtDlqyYuPOP5Jd_-XXNA12KlOPnNzrz3jm-5z8uQl6CjCcrVHUHJ75pGC8r9UOlS8cOgeXQB5dYP-fOWyo5CNadlozx1S2meCIxncbQRV1kCBi4KU";
-
- Y_UNIT_TEST(BlackboxTvmIdTest) {
- UNIT_ASSERT_VALUES_EQUAL("222", NBlackboxTvmId::Prod);
- UNIT_ASSERT_VALUES_EQUAL("224", NBlackboxTvmId::Test);
- UNIT_ASSERT_VALUES_EQUAL("223", NBlackboxTvmId::ProdYateam);
- UNIT_ASSERT_VALUES_EQUAL("225", NBlackboxTvmId::TestYateam);
- UNIT_ASSERT_VALUES_EQUAL("226", NBlackboxTvmId::Stress);
- UNIT_ASSERT_VALUES_EQUAL("239", NBlackboxTvmId::Mimino);
- }
-
- Y_UNIT_TEST(Case1Test) {
- TServiceContext context1(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- TServiceContext context2 = std::move(context1);
- TServiceContext context3(std::move(context2));
-
- TCheckedServiceTicket checkedTicket1 = context3.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket1.GetStatus());
- TCheckedServiceTicket checkedTicket2 = std::move(checkedTicket1);
- TCheckedServiceTicket checkedTicket3(std::move(checkedTicket2));
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket3.GetStatus());
- }
-
- Y_UNIT_TEST(ContextExceptionsTest) {
- UNIT_ASSERT_EXCEPTION(TServiceContext(SECRET, OUR_ID, MALFORMED_TVM_KEYS), TMalformedTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TServiceContext(SECRET, OUR_ID, EMPTY_TVM_KEYS), TEmptyTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TServiceContext(MALFORMED_TVM_SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS), TMalformedTvmSecretException);
- }
-
- Y_UNIT_TEST(ContextSignTest) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_VALUES_EQUAL(
- "NsPTYak4Cfk-4vgau5lab3W4GPiTtb2etuj3y4MDPrk",
- context.SignCgiParamsForTvm(IntToString<10>(std::numeric_limits<time_t>::max()), "13,28", ""));
- }
-
- Y_UNIT_TEST(ContextSignExceptionTest) {
- TServiceContext context = TServiceContext::CheckingFactory(OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EXCEPTION(
- context.SignCgiParamsForTvm(IntToString<10>(std::numeric_limits<time_t>::max()), "13,28", ""),
- TMalformedTvmSecretException
- );
-
- context = TServiceContext::SigningFactory(SECRET);
- UNIT_ASSERT_NO_EXCEPTION(
- context.SignCgiParamsForTvm(IntToString<10>(std::numeric_limits<time_t>::max()), "13,28", "")
- );
- }
-
- Y_UNIT_TEST(ContextCheckExceptionTest) {
- TServiceContext context = TServiceContext::CheckingFactory(OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_NO_EXCEPTION(
- context.Check("ABCDE")
- );
-
- context = TServiceContext::SigningFactory(SECRET);
- UNIT_ASSERT_EXCEPTION(
- context.Check("ABCDE"),
- TEmptyTvmKeysException
- );
- }
-
-
- Y_UNIT_TEST(ContextTest) {
- TServiceContext context1(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- TServiceContext context2 = TServiceContext::CheckingFactory(OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- }
-
- Y_UNIT_TEST(Ticket1Test) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_EQUAL(SRC_ID, checkedTicket.GetSrc());
- UNIT_ASSERT_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;scope=bb:sess1;scope=bb:sess2;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket2Test) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_2);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;scope=bb:sess1;scope=bb:sess10;scope=bb:sess100;scope=bb:sess11;scope=bb:sess12;scope=bb:sess13;scope=bb:sess14;scope=bb:sess15;scope=bb:sess16;scope=bb:sess17;scope=bb:sess18;scope=bb:sess19;scope=bb:sess2;scope=bb:sess20;scope=bb:sess21;scope=bb:sess22;scope=bb:sess23;scope=bb:sess24;scope=bb:sess25;scope=bb:sess26;scope=bb:sess27;scope=bb:sess28;scope=bb:sess29;scope=bb:sess3;scope=bb:sess30;scope=bb:sess31;scope=bb:sess32;scope=bb:sess33;scope=bb:sess34;scope=bb:sess35;scope=bb:sess36;scope=bb:sess37;scope=bb:sess38;scope=bb:sess39;scope=bb:sess4;scope=bb:sess40;scope=bb:sess41;scope=bb:sess42;scope=bb:sess43;scope=bb:sess44;scope=bb:sess45;scope=bb:sess46;scope=bb:sess47;scope=bb:sess48;scope=bb:sess49;scope=bb:sess5;scope=bb:sess50;scope=bb:sess51;scope=bb:sess52;scope=bb:sess53;scope=bb:sess54;scope=bb:sess55;scope=bb:sess56;scope=bb:sess57;scope=bb:sess58;scope=bb:sess59;scope=bb:sess6;scope=bb:sess60;scope=bb:sess61;scope=bb:sess62;scope=bb:sess63;scope=bb:sess64;scope=bb:sess65;scope=bb:sess66;scope=bb:sess67;scope=bb:sess68;scope=bb:sess69;scope=bb:sess7;scope=bb:sess70;scope=bb:sess71;scope=bb:sess72;scope=bb:sess73;scope=bb:sess74;scope=bb:sess75;scope=bb:sess76;scope=bb:sess77;scope=bb:sess78;scope=bb:sess79;scope=bb:sess8;scope=bb:sess80;scope=bb:sess81;scope=bb:sess82;scope=bb:sess83;scope=bb:sess84;scope=bb:sess85;scope=bb:sess86;scope=bb:sess87;scope=bb:sess88;scope=bb:sess89;scope=bb:sess9;scope=bb:sess90;scope=bb:sess91;scope=bb:sess92;scope=bb:sess93;scope=bb:sess94;scope=bb:sess95;scope=bb:sess96;scope=bb:sess97;scope=bb:sess98;scope=bb:sess99;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket3Test) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_3);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(TicketCheckingTest) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto ticket = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, ticket.GetStatus());
- UNIT_ASSERT_EQUAL(SRC_ID, ticket.GetSrc());
- }
-
- Y_UNIT_TEST(TicketErrorsTest) {
- TServiceContext context(SECRET, NOT_OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket1 = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::InvalidDst, checkedTicket1.GetStatus());
-
- auto checkedTicket2 = context.Check(UNSUPPORTED_VERSION_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::UnsupportedVersion, checkedTicket2.GetStatus());
-
- auto checkedTicket3 = context.Check(EXPIRED_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket3.GetStatus());
- }
-
- Y_UNIT_TEST(TicketExceptionsTest) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(EXPIRED_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket.GetStatus());
-
- UNIT_ASSERT(!bool(checkedTicket));
- UNIT_ASSERT_EXCEPTION(checkedTicket.GetSrc(), TNotAllowedException);
- UNIT_ASSERT_NO_EXCEPTION(bool(checkedTicket));
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket.DebugInfo());
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket.GetStatus());
- }
-
- Y_UNIT_TEST(RemoveSignatureTest) {
- UNIT_ASSERT_VALUES_EQUAL("1:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("1:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("2:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("2:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("4:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("4:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3.serv.ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("3.serv.ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("asdxcbvfgdsgfasdfxczvdsgfxcdvbcbvf",
- NUtils::RemoveTicketSignature("asdxcbvfgdsgfasdfxczvdsgfxcdvbcbvf"));
- }
-
- Y_UNIT_TEST(ResetKeysTest) {
- TServiceContext context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- TCheckedServiceTicket checkedTicket = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- }
-}
-
-Y_UNIT_TEST_SUITE(PublicInterfaceUserTestSuite) {
- static const TString EMPTY_TVM_KEYS = "1:EpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQY";
- static const TString EXPIRED_USER_TICKET = "3:user:CA0QABokCgMIyAMKAgh7EMgDGghiYjpzZXNzMRoIYmI6c2VzczIgEigB:D0CmYVwWg91LDYejjeQ2UP8AeiA_mr1q1CUD_lfJ9zQSEYEOYGDTafg4Um2rwOOvQnsD1JHM4zHyMUJ6Jtp9GAm5pmhbXBBZqaCcJpyxLTEC8a81MhJFCCJRvu_G1FiAgRgB25gI3HIbkvHFUEqAIC_nANy7NFQnbKk2S-EQPGY";
- static const TString MALFORMED_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAEEpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQYSlQEKkgEIEBAAGocBMIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbNIJ";
- static const TString UNSUPPORTED_VERSION_USER_TICKET = "2:user:CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE:KJFv5EcXn9krYk19LCvlFrhMW-R4q8mKfXJXCd-RBVBgUQzCOR1Dx2FiOyU-BxUoIsaU0PiwTjbVY5I2onJDilge70Cl5zEPI9pfab2qwklACq_ZBUvD1tzrfNUr88otBGAziHASJWgyVDkhyQ3p7YbN38qpb0vGQrYNxlk4e2I";
- static const TString USER_TICKET_PROTOBUF = "CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE";
- static const TString VALID_USER_TICKET_1 = "3:user:CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE:KJFv5EcXn9krYk19LCvlFrhMW-R4q8mKfXJXCd-RBVBgUQzCOR1Dx2FiOyU-BxUoIsaU0PiwTjbVY5I2onJDilge70Cl5zEPI9pfab2qwklACq_ZBUvD1tzrfNUr88otBGAziHASJWgyVDkhyQ3p7YbN38qpb0vGQrYNxlk4e2I";
- static const TString VALID_USER_TICKET_2 = "3:user:CA0Q__________9_GhAKAwjIAwoCCHsQyAMgEigB:KRibGYTJUA2ns0Fn7VYqeMZ1-GdscB1o9pRzELyr7QJrJsfsE8Y_HoVvB8Npr-oalv6AXOpagSc8HpZjAQz8zKMAVE_tI0tL-9DEsHirpawEbpy7OWV7-k18o1m-RaDaKeTlIB45KHbBul1-9aeKkortBfbbXtz_Qy9r_mfFPiQ";
- static const TString VALID_USER_TICKET_3 = "3:user:CA0Q__________9_Go8bCgIIAAoCCAEKAggCCgIIAwoCCAQKAggFCgIIBgoCCAcKAggICgIICQoCCAoKAggLCgIIDAoCCA0KAggOCgIIDwoCCBAKAggRCgIIEgoCCBMKAggUCgIIFQoCCBYKAggXCgIIGAoCCBkKAggaCgIIGwoCCBwKAggdCgIIHgoCCB8KAgggCgIIIQoCCCIKAggjCgIIJAoCCCUKAggmCgIIJwoCCCgKAggpCgIIKgoCCCsKAggsCgIILQoCCC4KAggvCgIIMAoCCDEKAggyCgIIMwoCCDQKAgg1CgIINgoCCDcKAgg4CgIIOQoCCDoKAgg7CgIIPAoCCD0KAgg-CgIIPwoCCEAKAghBCgIIQgoCCEMKAghECgIIRQoCCEYKAghHCgIISAoCCEkKAghKCgIISwoCCEwKAghNCgIITgoCCE8KAghQCgIIUQoCCFIKAghTCgIIVAoCCFUKAghWCgIIVwoCCFgKAghZCgIIWgoCCFsKAghcCgIIXQoCCF4KAghfCgIIYAoCCGEKAghiCgIIYwoCCGQKAghlCgIIZgoCCGcKAghoCgIIaQoCCGoKAghrCgIIbAoCCG0KAghuCgIIbwoCCHAKAghxCgIIcgoCCHMKAgh0CgIIdQoCCHYKAgh3CgIIeAoCCHkKAgh6CgIIewoCCHwKAgh9CgIIfgoCCH8KAwiAAQoDCIEBCgMIggEKAwiDAQoDCIQBCgMIhQEKAwiGAQoDCIcBCgMIiAEKAwiJAQoDCIoBCgMIiwEKAwiMAQoDCI0BCgMIjgEKAwiPAQoDCJABCgMIkQEKAwiSAQoDCJMBCgMIlAEKAwiVAQoDCJYBCgMIlwEKAwiYAQoDCJkBCgMImgEKAwibAQoDCJwBCgMInQEKAwieAQoDCJ8BCgMIoAEKAwihAQoDCKIBCgMIowEKAwikAQoDCKUBCgMIpgEKAwinAQoDCKgBCgMIqQEKAwiqAQoDCKsBCgMIrAEKAwitAQoDCK4BCgMIrwEKAwiwAQoDCLEBCgMIsgEKAwizAQoDCLQBCgMItQEKAwi2AQoDCLcBCgMIuAEKAwi5AQoDCLoBCgMIuwEKAwi8AQoDCL0BCgMIvgEKAwi_AQoDCMABCgMIwQEKAwjCAQoDCMMBCgMIxAEKAwjFAQoDCMYBCgMIxwEKAwjIAQoDCMkBCgMIygEKAwjLAQoDCMwBCgMIzQEKAwjOAQoDCM8BCgMI0AEKAwjRAQoDCNIBCgMI0wEKAwjUAQoDCNUBCgMI1gEKAwjXAQoDCNgBCgMI2QEKAwjaAQoDCNsBCgMI3AEKAwjdAQoDCN4BCgMI3wEKAwjgAQoDCOEBCgMI4gEKAwjjAQoDCOQBCgMI5QEKAwjmAQoDCOcBCgMI6AEKAwjpAQoDCOoBCgMI6wEKAwjsAQoDCO0BCgMI7gEKAwjvAQoDCPABCgMI8QEKAwjyAQoDCPMBCgMI9AEKAwj1AQoDCPYBCgMI9wEKAwj4AQoDCPkBCgMI-gEKAwj7AQoDCPwBCgMI_QEKAwj-AQoDCP8BCgMIgAIKAwiBAgoDCIICCgMIgwIKAwiEAgoDCIUCCgMIhgIKAwiHAgoDCIgCCgMIiQIKAwiKAgoDCIsCCgMIjAIKAwiNAgoDCI4CCgMIjwIKAwiQAgoDCJECCgMIkgIKAwiTAgoDCJQCCgMIlQIKAwiWAgoDCJcCCgMImAIKAwiZAgoDCJoCCgMImwIKAwicAgoDCJ0CCgMIngIKAwifAgoDCKACCgMIoQIKAwiiAgoDCKMCCgMIpAIKAwilAgoDCKYCCgMIpwIKAwioAgoDCKkCCgMIqgIKAwirAgoDCKwCCgMIrQIKAwiuAgoDCK8CCgMIsAIKAwixAgoDCLICCgMIswIKAwi0AgoDCLUCCgMItgIKAwi3AgoDCLgCCgMIuQIKAwi6AgoDCLsCCgMIvAIKAwi9AgoDCL4CCgMIvwIKAwjAAgoDCMECCgMIwgIKAwjDAgoDCMQCCgMIxQIKAwjGAgoDCMcCCgMIyAIKAwjJAgoDCMoCCgMIywIKAwjMAgoDCM0CCgMIzgIKAwjPAgoDCNACCgMI0QIKAwjSAgoDCNMCCgMI1AIKAwjVAgoDCNYCCgMI1wIKAwjYAgoDCNkCCgMI2gIKAwjbAgoDCNwCCgMI3QIKAwjeAgoDCN8CCgMI4AIKAwjhAgoDCOICCgMI4wIKAwjkAgoDCOUCCgMI5gIKAwjnAgoDCOgCCgMI6QIKAwjqAgoDCOsCCgMI7AIKAwjtAgoDCO4CCgMI7wIKAwjwAgoDCPECCgMI8gIKAwjzAgoDCPQCCgMI9QIKAwj2AgoDCPcCCgMI-AIKAwj5AgoDCPoCCgMI-wIKAwj8AgoDCP0CCgMI_gIKAwj_AgoDCIADCgMIgQMKAwiCAwoDCIMDCgMIhAMKAwiFAwoDCIYDCgMIhwMKAwiIAwoDCIkDCgMIigMKAwiLAwoDCIwDCgMIjQMKAwiOAwoDCI8DCgMIkAMKAwiRAwoDCJIDCgMIkwMKAwiUAwoDCJUDCgMIlgMKAwiXAwoDCJgDCgMImQMKAwiaAwoDCJsDCgMInAMKAwidAwoDCJ4DCgMInwMKAwigAwoDCKEDCgMIogMKAwijAwoDCKQDCgMIpQMKAwimAwoDCKcDCgMIqAMKAwipAwoDCKoDCgMIqwMKAwisAwoDCK0DCgMIrgMKAwivAwoDCLADCgMIsQMKAwiyAwoDCLMDCgMItAMKAwi1AwoDCLYDCgMItwMKAwi4AwoDCLkDCgMIugMKAwi7AwoDCLwDCgMIvQMKAwi-AwoDCL8DCgMIwAMKAwjBAwoDCMIDCgMIwwMKAwjEAwoDCMUDCgMIxgMKAwjHAwoDCMgDCgMIyQMKAwjKAwoDCMsDCgMIzAMKAwjNAwoDCM4DCgMIzwMKAwjQAwoDCNEDCgMI0gMKAwjTAwoDCNQDCgMI1QMKAwjWAwoDCNcDCgMI2AMKAwjZAwoDCNoDCgMI2wMKAwjcAwoDCN0DCgMI3gMKAwjfAwoDCOADCgMI4QMKAwjiAwoDCOMDCgMI5AMKAwjlAwoDCOYDCgMI5wMKAwjoAwoDCOkDCgMI6gMKAwjrAwoDCOwDCgMI7QMKAwjuAwoDCO8DCgMI8AMKAwjxAwoDCPIDCgMI8wMQyAMaCGJiOnNlc3MxGgliYjpzZXNzMTAaCmJiOnNlc3MxMDAaCWJiOnNlc3MxMRoJYmI6c2VzczEyGgliYjpzZXNzMTMaCWJiOnNlc3MxNBoJYmI6c2VzczE1GgliYjpzZXNzMTYaCWJiOnNlc3MxNxoJYmI6c2VzczE4GgliYjpzZXNzMTkaCGJiOnNlc3MyGgliYjpzZXNzMjAaCWJiOnNlc3MyMRoJYmI6c2VzczIyGgliYjpzZXNzMjMaCWJiOnNlc3MyNBoJYmI6c2VzczI1GgliYjpzZXNzMjYaCWJiOnNlc3MyNxoJYmI6c2VzczI4GgliYjpzZXNzMjkaCGJiOnNlc3MzGgliYjpzZXNzMzAaCWJiOnNlc3MzMRoJYmI6c2VzczMyGgliYjpzZXNzMzMaCWJiOnNlc3MzNBoJYmI6c2VzczM1GgliYjpzZXNzMzYaCWJiOnNlc3MzNxoJYmI6c2VzczM4GgliYjpzZXNzMzkaCGJiOnNlc3M0GgliYjpzZXNzNDAaCWJiOnNlc3M0MRoJYmI6c2VzczQyGgliYjpzZXNzNDMaCWJiOnNlc3M0NBoJYmI6c2VzczQ1GgliYjpzZXNzNDYaCWJiOnNlc3M0NxoJYmI6c2VzczQ4GgliYjpzZXNzNDkaCGJiOnNlc3M1GgliYjpzZXNzNTAaCWJiOnNlc3M1MRoJYmI6c2VzczUyGgliYjpzZXNzNTMaCWJiOnNlc3M1NBoJYmI6c2VzczU1GgliYjpzZXNzNTYaCWJiOnNlc3M1NxoJYmI6c2VzczU4GgliYjpzZXNzNTkaCGJiOnNlc3M2GgliYjpzZXNzNjAaCWJiOnNlc3M2MRoJYmI6c2VzczYyGgliYjpzZXNzNjMaCWJiOnNlc3M2NBoJYmI6c2VzczY1GgliYjpzZXNzNjYaCWJiOnNlc3M2NxoJYmI6c2VzczY4GgliYjpzZXNzNjkaCGJiOnNlc3M3GgliYjpzZXNzNzAaCWJiOnNlc3M3MRoJYmI6c2VzczcyGgliYjpzZXNzNzMaCWJiOnNlc3M3NBoJYmI6c2Vzczc1GgliYjpzZXNzNzYaCWJiOnNlc3M3NxoJYmI6c2Vzczc4GgliYjpzZXNzNzkaCGJiOnNlc3M4GgliYjpzZXNzODAaCWJiOnNlc3M4MRoJYmI6c2VzczgyGgliYjpzZXNzODMaCWJiOnNlc3M4NBoJYmI6c2Vzczg1GgliYjpzZXNzODYaCWJiOnNlc3M4NxoJYmI6c2Vzczg4GgliYjpzZXNzODkaCGJiOnNlc3M5GgliYjpzZXNzOTAaCWJiOnNlc3M5MRoJYmI6c2VzczkyGgliYjpzZXNzOTMaCWJiOnNlc3M5NBoJYmI6c2Vzczk1GgliYjpzZXNzOTYaCWJiOnNlc3M5NxoJYmI6c2Vzczk4GgliYjpzZXNzOTkgEigB:CX8PIOrxJnQqFXl7wAsiHJ_1VGjoI-asNlCXb8SE8jtI2vdh9x6CqbAurSgIlAAEgotVP-nuUR38x_a9YJuXzmG5AvJ458apWQtODHIDIX6ZaIwMxjS02R7S5LNqXa0gAuU_R6bCWpZdWe2uLMkdpu5KHbDgW08g-uaP_nceDOk";
-
- Y_UNIT_TEST(Case1Test) {
- TUserContext context1(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
-
- TCheckedUserTicket checkedTicket1 = context1.Check("2:serv:CgYIDRCUkQYQDBgcIgdiYjpzZXNzIghiYjpzZXNzMg:ERmeH_yzC7K_QsoHTyw7llCsyExEz3CoEopPIuivA0ZAtTaFq_Pa0l9Fhhx_NX9WpOp2CPyY5cFc4PXhcO83jCB7-EGvHNxGN-j2NQalERzPiKqkDCO0Q5etLzSzrfTlvMz7sXDvELNBHyA0PkAQnbz4supY0l-0Q6JBYSEF3zOVMjjE-HeQIFL3ats3_PakaUMWRvgQQ88pVdYZqAtbDw9PlTla7ommygVZQjcfNFXV1pJKRgOCLs-YyCjOJHLKL04zYj0X6KsOCTUeqhj7ml96wLZ-g1X9tyOR2WAr2Ctq7wIEHwqhxOLgOSKqm05xH6Vi3E_hekf50oe2jPfKEA");
- UNIT_ASSERT_EQUAL(ETicketStatus::UnsupportedVersion, checkedTicket1.GetStatus());
- UNIT_ASSERT(!checkedTicket1);
-
- TUserContext context2 = std::move(context1);
- TUserContext context3(std::move(context2));
- TCheckedUserTicket checkedTicket2 = context3.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket2.GetStatus());
- TCheckedUserTicket checkedTicket3 = std::move(checkedTicket2);
- TCheckedUserTicket checkedTicket4(std::move(checkedTicket3));
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket4.GetStatus());
- }
-
- Y_UNIT_TEST(ContextTest) {
- TUserContext context(EBlackboxEnv::Prod, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- }
-
- Y_UNIT_TEST(ContextExceptionsTest) {
- UNIT_ASSERT_EXCEPTION(TUserContext(EBlackboxEnv::Prod, EMPTY_TVM_KEYS), TEmptyTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TUserContext(EBlackboxEnv::Prod, MALFORMED_TVM_KEYS), TMalformedTvmKeysException);
- }
-
- Y_UNIT_TEST(Ticket1Test) {
- TUserContext context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_EQUAL(TUids({456, 123}), checkedTicket.GetUids());
- UNIT_ASSERT_EQUAL(456, checkedTicket.GetDefaultUid());
- UNIT_ASSERT_EQUAL(TScopes({"bb:sess1", "bb:sess2"}), checkedTicket.GetScopes());
- UNIT_ASSERT(checkedTicket.HasScope("bb:sess1"));
- UNIT_ASSERT(checkedTicket.HasScope("bb:sess2"));
- UNIT_ASSERT(!checkedTicket.HasScope("bb:sess3"));
- UNIT_ASSERT_EQUAL("ticket_type=user;expiration_time=9223372036854775807;scope=bb:sess1;scope=bb:sess2;default_uid=456;uid=456;uid=123;env=Test;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket2Test) {
- TUserContext context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_2);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=user;expiration_time=9223372036854775807;default_uid=456;uid=456;uid=123;env=Test;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket3Test) {
- TUserContext context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_3);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=user;expiration_time=9223372036854775807;scope=bb:sess1;scope=bb:sess10;scope=bb:sess100;scope=bb:sess11;scope=bb:sess12;scope=bb:sess13;scope=bb:sess14;scope=bb:sess15;scope=bb:sess16;scope=bb:sess17;scope=bb:sess18;scope=bb:sess19;scope=bb:sess2;scope=bb:sess20;scope=bb:sess21;scope=bb:sess22;scope=bb:sess23;scope=bb:sess24;scope=bb:sess25;scope=bb:sess26;scope=bb:sess27;scope=bb:sess28;scope=bb:sess29;scope=bb:sess3;scope=bb:sess30;scope=bb:sess31;scope=bb:sess32;scope=bb:sess33;scope=bb:sess34;scope=bb:sess35;scope=bb:sess36;scope=bb:sess37;scope=bb:sess38;scope=bb:sess39;scope=bb:sess4;scope=bb:sess40;scope=bb:sess41;scope=bb:sess42;scope=bb:sess43;scope=bb:sess44;scope=bb:sess45;scope=bb:sess46;scope=bb:sess47;scope=bb:sess48;scope=bb:sess49;scope=bb:sess5;scope=bb:sess50;scope=bb:sess51;scope=bb:sess52;scope=bb:sess53;scope=bb:sess54;scope=bb:sess55;scope=bb:sess56;scope=bb:sess57;scope=bb:sess58;scope=bb:sess59;scope=bb:sess6;scope=bb:sess60;scope=bb:sess61;scope=bb:sess62;scope=bb:sess63;scope=bb:sess64;scope=bb:sess65;scope=bb:sess66;scope=bb:sess67;scope=bb:sess68;scope=bb:sess69;scope=bb:sess7;scope=bb:sess70;scope=bb:sess71;scope=bb:sess72;scope=bb:sess73;scope=bb:sess74;scope=bb:sess75;scope=bb:sess76;scope=bb:sess77;scope=bb:sess78;scope=bb:sess79;scope=bb:sess8;scope=bb:sess80;scope=bb:sess81;scope=bb:sess82;scope=bb:sess83;scope=bb:sess84;scope=bb:sess85;scope=bb:sess86;scope=bb:sess87;scope=bb:sess88;scope=bb:sess89;scope=bb:sess9;scope=bb:sess90;scope=bb:sess91;scope=bb:sess92;scope=bb:sess93;scope=bb:sess94;scope=bb:sess95;scope=bb:sess96;scope=bb:sess97;scope=bb:sess98;scope=bb:sess99;default_uid=456;uid=0;uid=1;uid=2;uid=3;uid=4;uid=5;uid=6;uid=7;uid=8;uid=9;uid=10;uid=11;uid=12;uid=13;uid=14;uid=15;uid=16;uid=17;uid=18;uid=19;uid=20;uid=21;uid=22;uid=23;uid=24;uid=25;uid=26;uid=27;uid=28;uid=29;uid=30;uid=31;uid=32;uid=33;uid=34;uid=35;uid=36;uid=37;uid=38;uid=39;uid=40;uid=41;uid=42;uid=43;uid=44;uid=45;uid=46;uid=47;uid=48;uid=49;uid=50;uid=51;uid=52;uid=53;uid=54;uid=55;uid=56;uid=57;uid=58;uid=59;uid=60;uid=61;uid=62;uid=63;uid=64;uid=65;uid=66;uid=67;uid=68;uid=69;uid=70;uid=71;uid=72;uid=73;uid=74;uid=75;uid=76;uid=77;uid=78;uid=79;uid=80;uid=81;uid=82;uid=83;uid=84;uid=85;uid=86;uid=87;uid=88;uid=89;uid=90;uid=91;uid=92;uid=93;uid=94;uid=95;uid=96;uid=97;uid=98;uid=99;uid=100;uid=101;uid=102;uid=103;uid=104;uid=105;uid=106;uid=107;uid=108;uid=109;uid=110;uid=111;uid=112;uid=113;uid=114;uid=115;uid=116;uid=117;uid=118;uid=119;uid=120;uid=121;uid=122;uid=123;uid=124;uid=125;uid=126;uid=127;uid=128;uid=129;uid=130;uid=131;uid=132;uid=133;uid=134;uid=135;uid=136;uid=137;uid=138;uid=139;uid=140;uid=141;uid=142;uid=143;uid=144;uid=145;uid=146;uid=147;uid=148;uid=149;uid=150;uid=151;uid=152;uid=153;uid=154;uid=155;uid=156;uid=157;uid=158;uid=159;uid=160;uid=161;uid=162;uid=163;uid=164;uid=165;uid=166;uid=167;uid=168;uid=169;uid=170;uid=171;uid=172;uid=173;uid=174;uid=175;uid=176;uid=177;uid=178;uid=179;uid=180;uid=181;uid=182;uid=183;uid=184;uid=185;uid=186;uid=187;uid=188;uid=189;uid=190;uid=191;uid=192;uid=193;uid=194;uid=195;uid=196;uid=197;uid=198;uid=199;uid=200;uid=201;uid=202;uid=203;uid=204;uid=205;uid=206;uid=207;uid=208;uid=209;uid=210;uid=211;uid=212;uid=213;uid=214;uid=215;uid=216;uid=217;uid=218;uid=219;uid=220;uid=221;uid=222;uid=223;uid=224;uid=225;uid=226;uid=227;uid=228;uid=229;uid=230;uid=231;uid=232;uid=233;uid=234;uid=235;uid=236;uid=237;uid=238;uid=239;uid=240;uid=241;uid=242;uid=243;uid=244;uid=245;uid=246;uid=247;uid=248;uid=249;uid=250;uid=251;uid=252;uid=253;uid=254;uid=255;uid=256;uid=257;uid=258;uid=259;uid=260;uid=261;uid=262;uid=263;uid=264;uid=265;uid=266;uid=267;uid=268;uid=269;uid=270;uid=271;uid=272;uid=273;uid=274;uid=275;uid=276;uid=277;uid=278;uid=279;uid=280;uid=281;uid=282;uid=283;uid=284;uid=285;uid=286;uid=287;uid=288;uid=289;uid=290;uid=291;uid=292;uid=293;uid=294;uid=295;uid=296;uid=297;uid=298;uid=299;uid=300;uid=301;uid=302;uid=303;uid=304;uid=305;uid=306;uid=307;uid=308;uid=309;uid=310;uid=311;uid=312;uid=313;uid=314;uid=315;uid=316;uid=317;uid=318;uid=319;uid=320;uid=321;uid=322;uid=323;uid=324;uid=325;uid=326;uid=327;uid=328;uid=329;uid=330;uid=331;uid=332;uid=333;uid=334;uid=335;uid=336;uid=337;uid=338;uid=339;uid=340;uid=341;uid=342;uid=343;uid=344;uid=345;uid=346;uid=347;uid=348;uid=349;uid=350;uid=351;uid=352;uid=353;uid=354;uid=355;uid=356;uid=357;uid=358;uid=359;uid=360;uid=361;uid=362;uid=363;uid=364;uid=365;uid=366;uid=367;uid=368;uid=369;uid=370;uid=371;uid=372;uid=373;uid=374;uid=375;uid=376;uid=377;uid=378;uid=379;uid=380;uid=381;uid=382;uid=383;uid=384;uid=385;uid=386;uid=387;uid=388;uid=389;uid=390;uid=391;uid=392;uid=393;uid=394;uid=395;uid=396;uid=397;uid=398;uid=399;uid=400;uid=401;uid=402;uid=403;uid=404;uid=405;uid=406;uid=407;uid=408;uid=409;uid=410;uid=411;uid=412;uid=413;uid=414;uid=415;uid=416;uid=417;uid=418;uid=419;uid=420;uid=421;uid=422;uid=423;uid=424;uid=425;uid=426;uid=427;uid=428;uid=429;uid=430;uid=431;uid=432;uid=433;uid=434;uid=435;uid=436;uid=437;uid=438;uid=439;uid=440;uid=441;uid=442;uid=443;uid=444;uid=445;uid=446;uid=447;uid=448;uid=449;uid=450;uid=451;uid=452;uid=453;uid=454;uid=455;uid=456;uid=457;uid=458;uid=459;uid=460;uid=461;uid=462;uid=463;uid=464;uid=465;uid=466;uid=467;uid=468;uid=469;uid=470;uid=471;uid=472;uid=473;uid=474;uid=475;uid=476;uid=477;uid=478;uid=479;uid=480;uid=481;uid=482;uid=483;uid=484;uid=485;uid=486;uid=487;uid=488;uid=489;uid=490;uid=491;uid=492;uid=493;uid=494;uid=495;uid=496;uid=497;uid=498;uid=499;env=Test;", checkedTicket.DebugInfo());
- }
-
- Y_UNIT_TEST(TicketErrorsTest) {
- TUserContext contextTest(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket1 = contextTest.Check(UNSUPPORTED_VERSION_USER_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::UnsupportedVersion, checkedTicket1.GetStatus());
-
- auto checkedTicket2 = contextTest.Check(EXPIRED_USER_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket2.GetStatus());
-
- TUserContext contextProd(EBlackboxEnv::Prod, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket3 = contextProd.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::InvalidBlackboxEnv, checkedTicket3.GetStatus());
- }
-
- Y_UNIT_TEST(TicketExceptionsTest) {
- TUserContext contextTest(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = contextTest.Check(EXPIRED_USER_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket.GetStatus());
-
- UNIT_ASSERT_EXCEPTION(checkedTicket.GetDefaultUid(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket.GetUids(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket.GetScopes(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket.HasScope(""), TNotAllowedException);
- UNIT_ASSERT_NO_EXCEPTION(bool(checkedTicket));
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket.DebugInfo());
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket.GetStatus());
- }
-
- Y_UNIT_TEST(ResetKeysTest) {
- TUserContext context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/service_ut.cpp b/library/cpp/tvmauth/src/ut/service_ut.cpp
deleted file mode 100644
index 5b6b5143bd..0000000000
--- a/library/cpp/tvmauth/src/ut/service_ut.cpp
+++ /dev/null
@@ -1,156 +0,0 @@
-#include <library/cpp/tvmauth/src/service_impl.h>
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/string/cast.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(ServiceTestSuite) {
- Y_UNIT_TEST_DECLARE(TicketProtoTest);
-}
-
-class TTestServiceTicketImpl: public TCheckedServiceTicket::TImpl {
- using TCheckedServiceTicket::TImpl::TImpl;
- Y_UNIT_TEST_FRIEND(ServiceTestSuite, TicketProtoTest);
-};
-
-Y_UNIT_TEST_SUITE_IMPLEMENTATION(ServiceTestSuite) {
- static const TString EMPTY_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAE";
- static const TString EXPIRED_SERVICE_TICKET = "3:serv:CBAQACIZCOUBEBwaCGJiOnNlc3MxGghiYjpzZXNzMg:IwfMNJYEqStY_SixwqJnyHOMCPR7-3HHk4uylB2oVRkthtezq-OOA7QizDvx7VABLs_iTlXuD1r5IjufNei_EiV145eaa3HIg4xCdJXCojMexf2UYJz8mF2b0YzFAy6_KWagU7xo13CyKAqzJuQf5MJcSUf0ecY9hVh36cJ51aw";
- static const TString MALFORMED_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAEEpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQYSlQEKkgEIEBAAGocBMIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbNIJ";
- static const TString MALFORMED_TVM_SECRET = "adcvxcv./-+";
- static const TTvmId NOT_OUR_ID = 27;
- static const TTvmId OUR_ID = 28;
- static const TString SECRET = "GRMJrKnj4fOVnvOqe-WyD1";
- static const TString SERVICE_TICKET_PROTOBUF = "CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My";
- static const TTvmId SRC_ID = 229;
- static const TString UNSUPPORTED_VERSION_SERVICE_TICKET = "2:serv:CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My:WUPx1cTf05fjD1exB35T5j2DCHWH1YaLJon_a4rN-D7JfXHK1Ai4wM4uSfboHD9xmGQH7extqtlEk1tCTCGm5qbRVloJwWzCZBXo3zKX6i1oBYP_89WcjCNPVe1e8jwGdLsnu6PpxL5cn0xCksiStILH5UmDR6xfkJdnmMG94o8";
- static const TString VALID_SERVICE_TICKET_1 = "3:serv:CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My:WUPx1cTf05fjD1exB35T5j2DCHWH1YaLJon_a4rN-D7JfXHK1Ai4wM4uSfboHD9xmGQH7extqtlEk1tCTCGm5qbRVloJwWzCZBXo3zKX6i1oBYP_89WcjCNPVe1e8jwGdLsnu6PpxL5cn0xCksiStILH5UmDR6xfkJdnmMG94o8";
- static const TString VALID_SERVICE_TICKET_2 = "3:serv:CBAQ__________9_IskICOUBEBwaCGJiOnNlc3MxGgliYjpzZXNzMTAaCmJiOnNlc3MxMDAaCWJiOnNlc3MxMRoJYmI6c2VzczEyGgliYjpzZXNzMTMaCWJiOnNlc3MxNBoJYmI6c2VzczE1GgliYjpzZXNzMTYaCWJiOnNlc3MxNxoJYmI6c2VzczE4GgliYjpzZXNzMTkaCGJiOnNlc3MyGgliYjpzZXNzMjAaCWJiOnNlc3MyMRoJYmI6c2VzczIyGgliYjpzZXNzMjMaCWJiOnNlc3MyNBoJYmI6c2VzczI1GgliYjpzZXNzMjYaCWJiOnNlc3MyNxoJYmI6c2VzczI4GgliYjpzZXNzMjkaCGJiOnNlc3MzGgliYjpzZXNzMzAaCWJiOnNlc3MzMRoJYmI6c2VzczMyGgliYjpzZXNzMzMaCWJiOnNlc3MzNBoJYmI6c2VzczM1GgliYjpzZXNzMzYaCWJiOnNlc3MzNxoJYmI6c2VzczM4GgliYjpzZXNzMzkaCGJiOnNlc3M0GgliYjpzZXNzNDAaCWJiOnNlc3M0MRoJYmI6c2VzczQyGgliYjpzZXNzNDMaCWJiOnNlc3M0NBoJYmI6c2VzczQ1GgliYjpzZXNzNDYaCWJiOnNlc3M0NxoJYmI6c2VzczQ4GgliYjpzZXNzNDkaCGJiOnNlc3M1GgliYjpzZXNzNTAaCWJiOnNlc3M1MRoJYmI6c2VzczUyGgliYjpzZXNzNTMaCWJiOnNlc3M1NBoJYmI6c2VzczU1GgliYjpzZXNzNTYaCWJiOnNlc3M1NxoJYmI6c2VzczU4GgliYjpzZXNzNTkaCGJiOnNlc3M2GgliYjpzZXNzNjAaCWJiOnNlc3M2MRoJYmI6c2VzczYyGgliYjpzZXNzNjMaCWJiOnNlc3M2NBoJYmI6c2VzczY1GgliYjpzZXNzNjYaCWJiOnNlc3M2NxoJYmI6c2VzczY4GgliYjpzZXNzNjkaCGJiOnNlc3M3GgliYjpzZXNzNzAaCWJiOnNlc3M3MRoJYmI6c2VzczcyGgliYjpzZXNzNzMaCWJiOnNlc3M3NBoJYmI6c2Vzczc1GgliYjpzZXNzNzYaCWJiOnNlc3M3NxoJYmI6c2Vzczc4GgliYjpzZXNzNzkaCGJiOnNlc3M4GgliYjpzZXNzODAaCWJiOnNlc3M4MRoJYmI6c2VzczgyGgliYjpzZXNzODMaCWJiOnNlc3M4NBoJYmI6c2Vzczg1GgliYjpzZXNzODYaCWJiOnNlc3M4NxoJYmI6c2Vzczg4GgliYjpzZXNzODkaCGJiOnNlc3M5GgliYjpzZXNzOTAaCWJiOnNlc3M5MRoJYmI6c2VzczkyGgliYjpzZXNzOTMaCWJiOnNlc3M5NBoJYmI6c2Vzczk1GgliYjpzZXNzOTYaCWJiOnNlc3M5NxoJYmI6c2Vzczk4GgliYjpzZXNzOTk:JYmABAVLM6y7_T4n1pRcwBfwDfzMV4JJ3cpbEG617zdGgKRZwL7MalsYn5bq1F2ibujMrsF9nzZf8l4s_e-Ivjkz_xu4KMzSp-pUh9V7XIF_smj0WHYpv6gOvWNuK8uIvlZTTKwtQX0qZOL9m-MEeZiHoQPKZGCfJ_qxMUp-J8I";
- static const TString VALID_SERVICE_TICKET_3 = "3:serv:CBAQ__________9_IgUI5QEQHA:Sd6tmA1CNy2Nf7XevC3x7zr2DrGNRmcl-TxUsDtDW2xI3YXyCxBltWeg0-KtDlqyYuPOP5Jd_-XXNA12KlOPnNzrz3jm-5z8uQl6CjCcrVHUHJ75pGC8r9UOlS8cOgeXQB5dYP-fOWyo5CNadlozx1S2meCIxncbQRV1kCBi4KU";
- static const TString VALID_SERVICE_TICKET_ISSUER = "3:serv:CBAQ__________9_IgsI5QEQHCDr1MT4Ag:Gu66XJT_nKnIRJjFy1561wFhIqkJItcSTGftLo7Yvi7i5wIdV-QuKT_-IMPpgjxnnGbt1Dy3Ys2TEoeJAb0TdaCYG1uy3vpoLONmTx9AenN5dx1HHf46cypLK5D3OdiTjxvqI9uGmSIKrSdRxU8gprpu5QiBDPZqVCWhM60FVSY";
-
- Y_UNIT_TEST(ContextExceptionsTest) {
- UNIT_ASSERT_EXCEPTION(TServiceContext::TImpl(SECRET, OUR_ID, MALFORMED_TVM_KEYS), TMalformedTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TServiceContext::TImpl(SECRET, OUR_ID, EMPTY_TVM_KEYS), TEmptyTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TServiceContext::TImpl(MALFORMED_TVM_SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS), TMalformedTvmSecretException);
- }
-
- Y_UNIT_TEST(ContextSignTest) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_VALUES_EQUAL(
- "NsPTYak4Cfk-4vgau5lab3W4GPiTtb2etuj3y4MDPrk",
- context.SignCgiParamsForTvm(IntToString<10>(std::numeric_limits<time_t>::max()), "13,28", ""));
- }
-
- Y_UNIT_TEST(Ticket1Test) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_EQUAL(std::numeric_limits<time_t>::max(), checkedTicket->GetExpirationTime());
- UNIT_ASSERT_EQUAL(SRC_ID, checkedTicket->GetSrc());
- UNIT_ASSERT_EQUAL(TScopes({"bb:sess1", "bb:sess2"}), checkedTicket->GetScopes());
- UNIT_ASSERT(checkedTicket->HasScope("bb:sess1"));
- UNIT_ASSERT(checkedTicket->HasScope("bb:sess2"));
- UNIT_ASSERT(!checkedTicket->HasScope("bb:sess3"));
- UNIT_ASSERT_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;scope=bb:sess1;scope=bb:sess2;", checkedTicket->DebugInfo());
- UNIT_ASSERT(!checkedTicket->GetIssuerUid());
- }
-
- Y_UNIT_TEST(Ticket2Test) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_2);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;scope=bb:sess1;scope=bb:sess10;scope=bb:sess100;scope=bb:sess11;scope=bb:sess12;scope=bb:sess13;scope=bb:sess14;scope=bb:sess15;scope=bb:sess16;scope=bb:sess17;scope=bb:sess18;scope=bb:sess19;scope=bb:sess2;scope=bb:sess20;scope=bb:sess21;scope=bb:sess22;scope=bb:sess23;scope=bb:sess24;scope=bb:sess25;scope=bb:sess26;scope=bb:sess27;scope=bb:sess28;scope=bb:sess29;scope=bb:sess3;scope=bb:sess30;scope=bb:sess31;scope=bb:sess32;scope=bb:sess33;scope=bb:sess34;scope=bb:sess35;scope=bb:sess36;scope=bb:sess37;scope=bb:sess38;scope=bb:sess39;scope=bb:sess4;scope=bb:sess40;scope=bb:sess41;scope=bb:sess42;scope=bb:sess43;scope=bb:sess44;scope=bb:sess45;scope=bb:sess46;scope=bb:sess47;scope=bb:sess48;scope=bb:sess49;scope=bb:sess5;scope=bb:sess50;scope=bb:sess51;scope=bb:sess52;scope=bb:sess53;scope=bb:sess54;scope=bb:sess55;scope=bb:sess56;scope=bb:sess57;scope=bb:sess58;scope=bb:sess59;scope=bb:sess6;scope=bb:sess60;scope=bb:sess61;scope=bb:sess62;scope=bb:sess63;scope=bb:sess64;scope=bb:sess65;scope=bb:sess66;scope=bb:sess67;scope=bb:sess68;scope=bb:sess69;scope=bb:sess7;scope=bb:sess70;scope=bb:sess71;scope=bb:sess72;scope=bb:sess73;scope=bb:sess74;scope=bb:sess75;scope=bb:sess76;scope=bb:sess77;scope=bb:sess78;scope=bb:sess79;scope=bb:sess8;scope=bb:sess80;scope=bb:sess81;scope=bb:sess82;scope=bb:sess83;scope=bb:sess84;scope=bb:sess85;scope=bb:sess86;scope=bb:sess87;scope=bb:sess88;scope=bb:sess89;scope=bb:sess9;scope=bb:sess90;scope=bb:sess91;scope=bb:sess92;scope=bb:sess93;scope=bb:sess94;scope=bb:sess95;scope=bb:sess96;scope=bb:sess97;scope=bb:sess98;scope=bb:sess99;", checkedTicket->DebugInfo());
- UNIT_ASSERT(!checkedTicket->GetIssuerUid());
- }
-
- Y_UNIT_TEST(Ticket3Test) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_3);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;", checkedTicket->DebugInfo());
- UNIT_ASSERT(!checkedTicket->GetIssuerUid());
- }
-
- Y_UNIT_TEST(TicketIssuerTest) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_ISSUER);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;expiration_time=9223372036854775807;src=229;dst=28;issuer_uid=789654123;",
- checkedTicket->DebugInfo());
- UNIT_ASSERT(checkedTicket->GetIssuerUid());
- UNIT_ASSERT_VALUES_EQUAL(789654123, *checkedTicket->GetIssuerUid());
- }
-
- Y_UNIT_TEST(TicketErrorsTest) {
- TServiceContext::TImpl context(SECRET, NOT_OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket1 = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::InvalidDst, checkedTicket1->GetStatus());
-
- auto checkedTicket2 = context.Check(UNSUPPORTED_VERSION_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::UnsupportedVersion, checkedTicket2->GetStatus());
-
- auto checkedTicket3 = context.Check(EXPIRED_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket3->GetStatus());
- }
-
- Y_UNIT_TEST(TicketExceptionTest) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
-
- auto checkedTicket = context.Check(EXPIRED_SERVICE_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket->GetStatus());
-
- UNIT_ASSERT_EXCEPTION(checkedTicket->GetScopes(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket->GetSrc(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket->HasScope(""), TNotAllowedException);
- UNIT_ASSERT_NO_EXCEPTION(bool(*checkedTicket));
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket->DebugInfo());
- }
-
- Y_UNIT_TEST(TicketProtoTest) {
- ticket2::Ticket protobufTicket;
- UNIT_ASSERT(protobufTicket.ParseFromString(NUtils::Base64url2bin(SERVICE_TICKET_PROTOBUF)));
- TTestServiceTicketImpl checkedTicket(ETicketStatus::Ok, std::move(protobufTicket));
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket.GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(std::numeric_limits<time_t>::max(), checkedTicket.GetExpirationTime());
- UNIT_ASSERT_EQUAL(SRC_ID, checkedTicket.GetSrc());
- }
-
- Y_UNIT_TEST(ResetKeysTest) {
- TServiceContext::TImpl context(SECRET, OUR_ID, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- context.ResetKeys(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_SERVICE_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- }
-
- Y_UNIT_TEST(CreateTicketForTests) {
- TCheckedServiceTicket t = NTvmAuth::NUnittest::CreateServiceTicket(ETicketStatus::Ok, 42);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, t.GetStatus());
- UNIT_ASSERT_EQUAL(42, t.GetSrc());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=serv;src=42;dst=100500;", t.DebugInfo());
- }
-
- Y_UNIT_TEST(CreateForTests) {
- auto t = TCheckedServiceTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 456, {});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(456, t->GetSrc());
- UNIT_ASSERT(!t->GetIssuerUid());
-
- t = TCheckedServiceTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 456, 100800);
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(456, t->GetSrc());
- UNIT_ASSERT(t->GetIssuerUid());
- UNIT_ASSERT_VALUES_EQUAL(*t->GetIssuerUid(), 100800);
-
- t = TCheckedServiceTicket::TImpl::CreateTicketForTests(ETicketStatus::Expired, 456, {});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Expired, t->GetStatus());
- UNIT_ASSERT_EXCEPTION_CONTAINS(t->GetSrc(), TNotAllowedException, "Method cannot be used in non-valid ticket");
- UNIT_ASSERT_EXCEPTION_CONTAINS(t->GetIssuerUid(), TNotAllowedException, "Method cannot be used in non-valid ticket");
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/user_ut.cpp b/library/cpp/tvmauth/src/ut/user_ut.cpp
deleted file mode 100644
index c040e94974..0000000000
--- a/library/cpp/tvmauth/src/ut/user_ut.cpp
+++ /dev/null
@@ -1,216 +0,0 @@
-#include <library/cpp/tvmauth/src/user_impl.h>
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/tvmauth/exception.h>
-#include <library/cpp/tvmauth/unittest.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(UserTestSuite) {
- Y_UNIT_TEST_DECLARE(TicketProtoTest);
-}
-
-class TTestUserTicketImpl: TCheckedUserTicket::TImpl {
- using TCheckedUserTicket::TImpl::TImpl;
- Y_UNIT_TEST_FRIEND(UserTestSuite, TicketProtoTest);
-};
-
-Y_UNIT_TEST_SUITE_IMPLEMENTATION(UserTestSuite) {
- static const TString EMPTY_TVM_KEYS = "1:EpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQY";
- static const TString EXPIRED_USER_TICKET = "3:user:CA0QABokCgMIyAMKAgh7EMgDGghiYjpzZXNzMRoIYmI6c2VzczIgEigB:D0CmYVwWg91LDYejjeQ2UP8AeiA_mr1q1CUD_lfJ9zQSEYEOYGDTafg4Um2rwOOvQnsD1JHM4zHyMUJ6Jtp9GAm5pmhbXBBZqaCcJpyxLTEC8a81MhJFCCJRvu_G1FiAgRgB25gI3HIbkvHFUEqAIC_nANy7NFQnbKk2S-EQPGY";
- static const TString MALFORMED_TVM_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAEEpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQYSlQEKkgEIEBAAGocBMIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbNIJ";
- static const TString UNSUPPORTED_VERSION_USER_TICKET = "2:user:CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE:KJFv5EcXn9krYk19LCvlFrhMW-R4q8mKfXJXCd-RBVBgUQzCOR1Dx2FiOyU-BxUoIsaU0PiwTjbVY5I2onJDilge70Cl5zEPI9pfab2qwklACq_ZBUvD1tzrfNUr88otBGAziHASJWgyVDkhyQ3p7YbN38qpb0vGQrYNxlk4e2I";
- static const TString USER_TICKET_PROTOBUF = "CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE";
- static const TString VALID_USER_TICKET_1 = "3:user:CA0Q__________9_GiQKAwjIAwoCCHsQyAMaCGJiOnNlc3MxGghiYjpzZXNzMiASKAE:KJFv5EcXn9krYk19LCvlFrhMW-R4q8mKfXJXCd-RBVBgUQzCOR1Dx2FiOyU-BxUoIsaU0PiwTjbVY5I2onJDilge70Cl5zEPI9pfab2qwklACq_ZBUvD1tzrfNUr88otBGAziHASJWgyVDkhyQ3p7YbN38qpb0vGQrYNxlk4e2I";
- static const TString VALID_USER_TICKET_2 = "3:user:CA0Q__________9_GhAKAwjIAwoCCHsQyAMgEigB:KRibGYTJUA2ns0Fn7VYqeMZ1-GdscB1o9pRzELyr7QJrJsfsE8Y_HoVvB8Npr-oalv6AXOpagSc8HpZjAQz8zKMAVE_tI0tL-9DEsHirpawEbpy7OWV7-k18o1m-RaDaKeTlIB45KHbBul1-9aeKkortBfbbXtz_Qy9r_mfFPiQ";
- static const TString VALID_USER_TICKET_3 = "3:user:CA0Q__________9_Go8bCgIIAAoCCAEKAggCCgIIAwoCCAQKAggFCgIIBgoCCAcKAggICgIICQoCCAoKAggLCgIIDAoCCA0KAggOCgIIDwoCCBAKAggRCgIIEgoCCBMKAggUCgIIFQoCCBYKAggXCgIIGAoCCBkKAggaCgIIGwoCCBwKAggdCgIIHgoCCB8KAgggCgIIIQoCCCIKAggjCgIIJAoCCCUKAggmCgIIJwoCCCgKAggpCgIIKgoCCCsKAggsCgIILQoCCC4KAggvCgIIMAoCCDEKAggyCgIIMwoCCDQKAgg1CgIINgoCCDcKAgg4CgIIOQoCCDoKAgg7CgIIPAoCCD0KAgg-CgIIPwoCCEAKAghBCgIIQgoCCEMKAghECgIIRQoCCEYKAghHCgIISAoCCEkKAghKCgIISwoCCEwKAghNCgIITgoCCE8KAghQCgIIUQoCCFIKAghTCgIIVAoCCFUKAghWCgIIVwoCCFgKAghZCgIIWgoCCFsKAghcCgIIXQoCCF4KAghfCgIIYAoCCGEKAghiCgIIYwoCCGQKAghlCgIIZgoCCGcKAghoCgIIaQoCCGoKAghrCgIIbAoCCG0KAghuCgIIbwoCCHAKAghxCgIIcgoCCHMKAgh0CgIIdQoCCHYKAgh3CgIIeAoCCHkKAgh6CgIIewoCCHwKAgh9CgIIfgoCCH8KAwiAAQoDCIEBCgMIggEKAwiDAQoDCIQBCgMIhQEKAwiGAQoDCIcBCgMIiAEKAwiJAQoDCIoBCgMIiwEKAwiMAQoDCI0BCgMIjgEKAwiPAQoDCJABCgMIkQEKAwiSAQoDCJMBCgMIlAEKAwiVAQoDCJYBCgMIlwEKAwiYAQoDCJkBCgMImgEKAwibAQoDCJwBCgMInQEKAwieAQoDCJ8BCgMIoAEKAwihAQoDCKIBCgMIowEKAwikAQoDCKUBCgMIpgEKAwinAQoDCKgBCgMIqQEKAwiqAQoDCKsBCgMIrAEKAwitAQoDCK4BCgMIrwEKAwiwAQoDCLEBCgMIsgEKAwizAQoDCLQBCgMItQEKAwi2AQoDCLcBCgMIuAEKAwi5AQoDCLoBCgMIuwEKAwi8AQoDCL0BCgMIvgEKAwi_AQoDCMABCgMIwQEKAwjCAQoDCMMBCgMIxAEKAwjFAQoDCMYBCgMIxwEKAwjIAQoDCMkBCgMIygEKAwjLAQoDCMwBCgMIzQEKAwjOAQoDCM8BCgMI0AEKAwjRAQoDCNIBCgMI0wEKAwjUAQoDCNUBCgMI1gEKAwjXAQoDCNgBCgMI2QEKAwjaAQoDCNsBCgMI3AEKAwjdAQoDCN4BCgMI3wEKAwjgAQoDCOEBCgMI4gEKAwjjAQoDCOQBCgMI5QEKAwjmAQoDCOcBCgMI6AEKAwjpAQoDCOoBCgMI6wEKAwjsAQoDCO0BCgMI7gEKAwjvAQoDCPABCgMI8QEKAwjyAQoDCPMBCgMI9AEKAwj1AQoDCPYBCgMI9wEKAwj4AQoDCPkBCgMI-gEKAwj7AQoDCPwBCgMI_QEKAwj-AQoDCP8BCgMIgAIKAwiBAgoDCIICCgMIgwIKAwiEAgoDCIUCCgMIhgIKAwiHAgoDCIgCCgMIiQIKAwiKAgoDCIsCCgMIjAIKAwiNAgoDCI4CCgMIjwIKAwiQAgoDCJECCgMIkgIKAwiTAgoDCJQCCgMIlQIKAwiWAgoDCJcCCgMImAIKAwiZAgoDCJoCCgMImwIKAwicAgoDCJ0CCgMIngIKAwifAgoDCKACCgMIoQIKAwiiAgoDCKMCCgMIpAIKAwilAgoDCKYCCgMIpwIKAwioAgoDCKkCCgMIqgIKAwirAgoDCKwCCgMIrQIKAwiuAgoDCK8CCgMIsAIKAwixAgoDCLICCgMIswIKAwi0AgoDCLUCCgMItgIKAwi3AgoDCLgCCgMIuQIKAwi6AgoDCLsCCgMIvAIKAwi9AgoDCL4CCgMIvwIKAwjAAgoDCMECCgMIwgIKAwjDAgoDCMQCCgMIxQIKAwjGAgoDCMcCCgMIyAIKAwjJAgoDCMoCCgMIywIKAwjMAgoDCM0CCgMIzgIKAwjPAgoDCNACCgMI0QIKAwjSAgoDCNMCCgMI1AIKAwjVAgoDCNYCCgMI1wIKAwjYAgoDCNkCCgMI2gIKAwjbAgoDCNwCCgMI3QIKAwjeAgoDCN8CCgMI4AIKAwjhAgoDCOICCgMI4wIKAwjkAgoDCOUCCgMI5gIKAwjnAgoDCOgCCgMI6QIKAwjqAgoDCOsCCgMI7AIKAwjtAgoDCO4CCgMI7wIKAwjwAgoDCPECCgMI8gIKAwjzAgoDCPQCCgMI9QIKAwj2AgoDCPcCCgMI-AIKAwj5AgoDCPoCCgMI-wIKAwj8AgoDCP0CCgMI_gIKAwj_AgoDCIADCgMIgQMKAwiCAwoDCIMDCgMIhAMKAwiFAwoDCIYDCgMIhwMKAwiIAwoDCIkDCgMIigMKAwiLAwoDCIwDCgMIjQMKAwiOAwoDCI8DCgMIkAMKAwiRAwoDCJIDCgMIkwMKAwiUAwoDCJUDCgMIlgMKAwiXAwoDCJgDCgMImQMKAwiaAwoDCJsDCgMInAMKAwidAwoDCJ4DCgMInwMKAwigAwoDCKEDCgMIogMKAwijAwoDCKQDCgMIpQMKAwimAwoDCKcDCgMIqAMKAwipAwoDCKoDCgMIqwMKAwisAwoDCK0DCgMIrgMKAwivAwoDCLADCgMIsQMKAwiyAwoDCLMDCgMItAMKAwi1AwoDCLYDCgMItwMKAwi4AwoDCLkDCgMIugMKAwi7AwoDCLwDCgMIvQMKAwi-AwoDCL8DCgMIwAMKAwjBAwoDCMIDCgMIwwMKAwjEAwoDCMUDCgMIxgMKAwjHAwoDCMgDCgMIyQMKAwjKAwoDCMsDCgMIzAMKAwjNAwoDCM4DCgMIzwMKAwjQAwoDCNEDCgMI0gMKAwjTAwoDCNQDCgMI1QMKAwjWAwoDCNcDCgMI2AMKAwjZAwoDCNoDCgMI2wMKAwjcAwoDCN0DCgMI3gMKAwjfAwoDCOADCgMI4QMKAwjiAwoDCOMDCgMI5AMKAwjlAwoDCOYDCgMI5wMKAwjoAwoDCOkDCgMI6gMKAwjrAwoDCOwDCgMI7QMKAwjuAwoDCO8DCgMI8AMKAwjxAwoDCPIDCgMI8wMQyAMaCGJiOnNlc3MxGgliYjpzZXNzMTAaCmJiOnNlc3MxMDAaCWJiOnNlc3MxMRoJYmI6c2VzczEyGgliYjpzZXNzMTMaCWJiOnNlc3MxNBoJYmI6c2VzczE1GgliYjpzZXNzMTYaCWJiOnNlc3MxNxoJYmI6c2VzczE4GgliYjpzZXNzMTkaCGJiOnNlc3MyGgliYjpzZXNzMjAaCWJiOnNlc3MyMRoJYmI6c2VzczIyGgliYjpzZXNzMjMaCWJiOnNlc3MyNBoJYmI6c2VzczI1GgliYjpzZXNzMjYaCWJiOnNlc3MyNxoJYmI6c2VzczI4GgliYjpzZXNzMjkaCGJiOnNlc3MzGgliYjpzZXNzMzAaCWJiOnNlc3MzMRoJYmI6c2VzczMyGgliYjpzZXNzMzMaCWJiOnNlc3MzNBoJYmI6c2VzczM1GgliYjpzZXNzMzYaCWJiOnNlc3MzNxoJYmI6c2VzczM4GgliYjpzZXNzMzkaCGJiOnNlc3M0GgliYjpzZXNzNDAaCWJiOnNlc3M0MRoJYmI6c2VzczQyGgliYjpzZXNzNDMaCWJiOnNlc3M0NBoJYmI6c2VzczQ1GgliYjpzZXNzNDYaCWJiOnNlc3M0NxoJYmI6c2VzczQ4GgliYjpzZXNzNDkaCGJiOnNlc3M1GgliYjpzZXNzNTAaCWJiOnNlc3M1MRoJYmI6c2VzczUyGgliYjpzZXNzNTMaCWJiOnNlc3M1NBoJYmI6c2VzczU1GgliYjpzZXNzNTYaCWJiOnNlc3M1NxoJYmI6c2VzczU4GgliYjpzZXNzNTkaCGJiOnNlc3M2GgliYjpzZXNzNjAaCWJiOnNlc3M2MRoJYmI6c2VzczYyGgliYjpzZXNzNjMaCWJiOnNlc3M2NBoJYmI6c2VzczY1GgliYjpzZXNzNjYaCWJiOnNlc3M2NxoJYmI6c2VzczY4GgliYjpzZXNzNjkaCGJiOnNlc3M3GgliYjpzZXNzNzAaCWJiOnNlc3M3MRoJYmI6c2VzczcyGgliYjpzZXNzNzMaCWJiOnNlc3M3NBoJYmI6c2Vzczc1GgliYjpzZXNzNzYaCWJiOnNlc3M3NxoJYmI6c2Vzczc4GgliYjpzZXNzNzkaCGJiOnNlc3M4GgliYjpzZXNzODAaCWJiOnNlc3M4MRoJYmI6c2VzczgyGgliYjpzZXNzODMaCWJiOnNlc3M4NBoJYmI6c2Vzczg1GgliYjpzZXNzODYaCWJiOnNlc3M4NxoJYmI6c2Vzczg4GgliYjpzZXNzODkaCGJiOnNlc3M5GgliYjpzZXNzOTAaCWJiOnNlc3M5MRoJYmI6c2VzczkyGgliYjpzZXNzOTMaCWJiOnNlc3M5NBoJYmI6c2Vzczk1GgliYjpzZXNzOTYaCWJiOnNlc3M5NxoJYmI6c2Vzczk4GgliYjpzZXNzOTkgEigB:CX8PIOrxJnQqFXl7wAsiHJ_1VGjoI-asNlCXb8SE8jtI2vdh9x6CqbAurSgIlAAEgotVP-nuUR38x_a9YJuXzmG5AvJ458apWQtODHIDIX6ZaIwMxjS02R7S5LNqXa0gAuU_R6bCWpZdWe2uLMkdpu5KHbDgW08g-uaP_nceDOk";
-
- Y_UNIT_TEST(ContextText) {
- TUserContext::TImpl context(EBlackboxEnv::Prod, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(2, context.GetKeys().size());
- UNIT_ASSERT_NO_EXCEPTION(context.ResetKeys(NUnittest::TVMKNIFE_PUBLIC_KEYS));
- UNIT_ASSERT_EQUAL(2, context.GetKeys().size());
- }
-
- Y_UNIT_TEST(ContextEnvTest) {
- TUserContext::TImpl p(EBlackboxEnv::Prod, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(2, p.GetKeys().size());
- UNIT_ASSERT(p.IsAllowed(tvm_keys::Prod));
- UNIT_ASSERT(!p.IsAllowed(tvm_keys::ProdYateam));
- UNIT_ASSERT(!p.IsAllowed(tvm_keys::Test));
- UNIT_ASSERT(!p.IsAllowed(tvm_keys::TestYateam));
- UNIT_ASSERT(!p.IsAllowed(tvm_keys::Stress));
-
- TUserContext::TImpl pt(EBlackboxEnv::ProdYateam, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(2, pt.GetKeys().size());
- UNIT_ASSERT(!pt.IsAllowed(tvm_keys::Prod));
- UNIT_ASSERT(pt.IsAllowed(tvm_keys::ProdYateam));
- UNIT_ASSERT(!pt.IsAllowed(tvm_keys::Test));
- UNIT_ASSERT(!pt.IsAllowed(tvm_keys::TestYateam));
- UNIT_ASSERT(!pt.IsAllowed(tvm_keys::Stress));
-
- TUserContext::TImpl t(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(2, t.GetKeys().size());
- UNIT_ASSERT(!t.IsAllowed(tvm_keys::Prod));
- UNIT_ASSERT(!t.IsAllowed(tvm_keys::ProdYateam));
- UNIT_ASSERT(t.IsAllowed(tvm_keys::Test));
- UNIT_ASSERT(!t.IsAllowed(tvm_keys::TestYateam));
- UNIT_ASSERT(!t.IsAllowed(tvm_keys::Stress));
-
- TUserContext::TImpl tt(EBlackboxEnv::TestYateam, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(2, tt.GetKeys().size());
- UNIT_ASSERT(!tt.IsAllowed(tvm_keys::Prod));
- UNIT_ASSERT(!tt.IsAllowed(tvm_keys::ProdYateam));
- UNIT_ASSERT(!tt.IsAllowed(tvm_keys::Test));
- UNIT_ASSERT(tt.IsAllowed(tvm_keys::TestYateam));
- UNIT_ASSERT(!tt.IsAllowed(tvm_keys::Stress));
-
- TUserContext::TImpl s(EBlackboxEnv::Stress, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- UNIT_ASSERT_EQUAL(4, s.GetKeys().size());
- UNIT_ASSERT(s.IsAllowed(tvm_keys::Prod));
- UNIT_ASSERT(!s.IsAllowed(tvm_keys::ProdYateam));
- UNIT_ASSERT(!s.IsAllowed(tvm_keys::Test));
- UNIT_ASSERT(!s.IsAllowed(tvm_keys::TestYateam));
- UNIT_ASSERT(s.IsAllowed(tvm_keys::Stress));
- }
-
- Y_UNIT_TEST(ContextExceptionsText) {
- UNIT_ASSERT_EXCEPTION(TUserContext::TImpl(EBlackboxEnv::Prod, EMPTY_TVM_KEYS), TEmptyTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TUserContext::TImpl(EBlackboxEnv::Prod, MALFORMED_TVM_KEYS), TMalformedTvmKeysException);
- UNIT_ASSERT_EXCEPTION(TUserContext::TImpl(EBlackboxEnv::Prod, "adcvxcv./-+"), TMalformedTvmKeysException);
- }
-
- Y_UNIT_TEST(Ticket1Test) {
- TUserContext::TImpl context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_EQUAL(std::numeric_limits<time_t>::max(), checkedTicket->GetExpirationTime());
- UNIT_ASSERT_EQUAL(TUids({456, 123}), checkedTicket->GetUids());
- UNIT_ASSERT_EQUAL(456, checkedTicket->GetDefaultUid());
- UNIT_ASSERT_EQUAL(TScopes({"bb:sess1", "bb:sess2"}), checkedTicket->GetScopes());
- UNIT_ASSERT(checkedTicket->HasScope("bb:sess1"));
- UNIT_ASSERT(checkedTicket->HasScope("bb:sess2"));
- UNIT_ASSERT(!checkedTicket->HasScope("bb:sess3"));
- UNIT_ASSERT_EQUAL("ticket_type=user;expiration_time=9223372036854775807;scope=bb:sess1;scope=bb:sess2;default_uid=456;uid=456;uid=123;env=Test;", checkedTicket->DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket2Test) {
- TUserContext::TImpl context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_2);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=user;expiration_time=9223372036854775807;default_uid=456;uid=456;uid=123;env=Test;", checkedTicket->DebugInfo());
- }
-
- Y_UNIT_TEST(Ticket3Test) {
- TUserContext::TImpl context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_3);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=user;expiration_time=9223372036854775807;scope=bb:sess1;scope=bb:sess10;scope=bb:sess100;scope=bb:sess11;scope=bb:sess12;scope=bb:sess13;scope=bb:sess14;scope=bb:sess15;scope=bb:sess16;scope=bb:sess17;scope=bb:sess18;scope=bb:sess19;scope=bb:sess2;scope=bb:sess20;scope=bb:sess21;scope=bb:sess22;scope=bb:sess23;scope=bb:sess24;scope=bb:sess25;scope=bb:sess26;scope=bb:sess27;scope=bb:sess28;scope=bb:sess29;scope=bb:sess3;scope=bb:sess30;scope=bb:sess31;scope=bb:sess32;scope=bb:sess33;scope=bb:sess34;scope=bb:sess35;scope=bb:sess36;scope=bb:sess37;scope=bb:sess38;scope=bb:sess39;scope=bb:sess4;scope=bb:sess40;scope=bb:sess41;scope=bb:sess42;scope=bb:sess43;scope=bb:sess44;scope=bb:sess45;scope=bb:sess46;scope=bb:sess47;scope=bb:sess48;scope=bb:sess49;scope=bb:sess5;scope=bb:sess50;scope=bb:sess51;scope=bb:sess52;scope=bb:sess53;scope=bb:sess54;scope=bb:sess55;scope=bb:sess56;scope=bb:sess57;scope=bb:sess58;scope=bb:sess59;scope=bb:sess6;scope=bb:sess60;scope=bb:sess61;scope=bb:sess62;scope=bb:sess63;scope=bb:sess64;scope=bb:sess65;scope=bb:sess66;scope=bb:sess67;scope=bb:sess68;scope=bb:sess69;scope=bb:sess7;scope=bb:sess70;scope=bb:sess71;scope=bb:sess72;scope=bb:sess73;scope=bb:sess74;scope=bb:sess75;scope=bb:sess76;scope=bb:sess77;scope=bb:sess78;scope=bb:sess79;scope=bb:sess8;scope=bb:sess80;scope=bb:sess81;scope=bb:sess82;scope=bb:sess83;scope=bb:sess84;scope=bb:sess85;scope=bb:sess86;scope=bb:sess87;scope=bb:sess88;scope=bb:sess89;scope=bb:sess9;scope=bb:sess90;scope=bb:sess91;scope=bb:sess92;scope=bb:sess93;scope=bb:sess94;scope=bb:sess95;scope=bb:sess96;scope=bb:sess97;scope=bb:sess98;scope=bb:sess99;default_uid=456;uid=0;uid=1;uid=2;uid=3;uid=4;uid=5;uid=6;uid=7;uid=8;uid=9;uid=10;uid=11;uid=12;uid=13;uid=14;uid=15;uid=16;uid=17;uid=18;uid=19;uid=20;uid=21;uid=22;uid=23;uid=24;uid=25;uid=26;uid=27;uid=28;uid=29;uid=30;uid=31;uid=32;uid=33;uid=34;uid=35;uid=36;uid=37;uid=38;uid=39;uid=40;uid=41;uid=42;uid=43;uid=44;uid=45;uid=46;uid=47;uid=48;uid=49;uid=50;uid=51;uid=52;uid=53;uid=54;uid=55;uid=56;uid=57;uid=58;uid=59;uid=60;uid=61;uid=62;uid=63;uid=64;uid=65;uid=66;uid=67;uid=68;uid=69;uid=70;uid=71;uid=72;uid=73;uid=74;uid=75;uid=76;uid=77;uid=78;uid=79;uid=80;uid=81;uid=82;uid=83;uid=84;uid=85;uid=86;uid=87;uid=88;uid=89;uid=90;uid=91;uid=92;uid=93;uid=94;uid=95;uid=96;uid=97;uid=98;uid=99;uid=100;uid=101;uid=102;uid=103;uid=104;uid=105;uid=106;uid=107;uid=108;uid=109;uid=110;uid=111;uid=112;uid=113;uid=114;uid=115;uid=116;uid=117;uid=118;uid=119;uid=120;uid=121;uid=122;uid=123;uid=124;uid=125;uid=126;uid=127;uid=128;uid=129;uid=130;uid=131;uid=132;uid=133;uid=134;uid=135;uid=136;uid=137;uid=138;uid=139;uid=140;uid=141;uid=142;uid=143;uid=144;uid=145;uid=146;uid=147;uid=148;uid=149;uid=150;uid=151;uid=152;uid=153;uid=154;uid=155;uid=156;uid=157;uid=158;uid=159;uid=160;uid=161;uid=162;uid=163;uid=164;uid=165;uid=166;uid=167;uid=168;uid=169;uid=170;uid=171;uid=172;uid=173;uid=174;uid=175;uid=176;uid=177;uid=178;uid=179;uid=180;uid=181;uid=182;uid=183;uid=184;uid=185;uid=186;uid=187;uid=188;uid=189;uid=190;uid=191;uid=192;uid=193;uid=194;uid=195;uid=196;uid=197;uid=198;uid=199;uid=200;uid=201;uid=202;uid=203;uid=204;uid=205;uid=206;uid=207;uid=208;uid=209;uid=210;uid=211;uid=212;uid=213;uid=214;uid=215;uid=216;uid=217;uid=218;uid=219;uid=220;uid=221;uid=222;uid=223;uid=224;uid=225;uid=226;uid=227;uid=228;uid=229;uid=230;uid=231;uid=232;uid=233;uid=234;uid=235;uid=236;uid=237;uid=238;uid=239;uid=240;uid=241;uid=242;uid=243;uid=244;uid=245;uid=246;uid=247;uid=248;uid=249;uid=250;uid=251;uid=252;uid=253;uid=254;uid=255;uid=256;uid=257;uid=258;uid=259;uid=260;uid=261;uid=262;uid=263;uid=264;uid=265;uid=266;uid=267;uid=268;uid=269;uid=270;uid=271;uid=272;uid=273;uid=274;uid=275;uid=276;uid=277;uid=278;uid=279;uid=280;uid=281;uid=282;uid=283;uid=284;uid=285;uid=286;uid=287;uid=288;uid=289;uid=290;uid=291;uid=292;uid=293;uid=294;uid=295;uid=296;uid=297;uid=298;uid=299;uid=300;uid=301;uid=302;uid=303;uid=304;uid=305;uid=306;uid=307;uid=308;uid=309;uid=310;uid=311;uid=312;uid=313;uid=314;uid=315;uid=316;uid=317;uid=318;uid=319;uid=320;uid=321;uid=322;uid=323;uid=324;uid=325;uid=326;uid=327;uid=328;uid=329;uid=330;uid=331;uid=332;uid=333;uid=334;uid=335;uid=336;uid=337;uid=338;uid=339;uid=340;uid=341;uid=342;uid=343;uid=344;uid=345;uid=346;uid=347;uid=348;uid=349;uid=350;uid=351;uid=352;uid=353;uid=354;uid=355;uid=356;uid=357;uid=358;uid=359;uid=360;uid=361;uid=362;uid=363;uid=364;uid=365;uid=366;uid=367;uid=368;uid=369;uid=370;uid=371;uid=372;uid=373;uid=374;uid=375;uid=376;uid=377;uid=378;uid=379;uid=380;uid=381;uid=382;uid=383;uid=384;uid=385;uid=386;uid=387;uid=388;uid=389;uid=390;uid=391;uid=392;uid=393;uid=394;uid=395;uid=396;uid=397;uid=398;uid=399;uid=400;uid=401;uid=402;uid=403;uid=404;uid=405;uid=406;uid=407;uid=408;uid=409;uid=410;uid=411;uid=412;uid=413;uid=414;uid=415;uid=416;uid=417;uid=418;uid=419;uid=420;uid=421;uid=422;uid=423;uid=424;uid=425;uid=426;uid=427;uid=428;uid=429;uid=430;uid=431;uid=432;uid=433;uid=434;uid=435;uid=436;uid=437;uid=438;uid=439;uid=440;uid=441;uid=442;uid=443;uid=444;uid=445;uid=446;uid=447;uid=448;uid=449;uid=450;uid=451;uid=452;uid=453;uid=454;uid=455;uid=456;uid=457;uid=458;uid=459;uid=460;uid=461;uid=462;uid=463;uid=464;uid=465;uid=466;uid=467;uid=468;uid=469;uid=470;uid=471;uid=472;uid=473;uid=474;uid=475;uid=476;uid=477;uid=478;uid=479;uid=480;uid=481;uid=482;uid=483;uid=484;uid=485;uid=486;uid=487;uid=488;uid=489;uid=490;uid=491;uid=492;uid=493;uid=494;uid=495;uid=496;uid=497;uid=498;uid=499;env=Test;", checkedTicket->DebugInfo());
- }
-
- Y_UNIT_TEST(TicketExceptionsTest) {
- TUserContext::TImpl contextTest(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket1 = contextTest.Check(UNSUPPORTED_VERSION_USER_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::UnsupportedVersion, checkedTicket1->GetStatus());
-
- auto checkedTicket2 = contextTest.Check(EXPIRED_USER_TICKET);
- UNIT_ASSERT_EQUAL(ETicketStatus::Expired, checkedTicket2->GetStatus());
-
- TUserContext::TImpl contextProd(EBlackboxEnv::Prod, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket3 = contextProd.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::InvalidBlackboxEnv, checkedTicket3->GetStatus());
-
- UNIT_ASSERT_EXCEPTION(checkedTicket3->GetDefaultUid(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket3->GetUids(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket3->GetScopes(), TNotAllowedException);
- UNIT_ASSERT_EXCEPTION(checkedTicket3->HasScope(""), TNotAllowedException);
- UNIT_ASSERT_NO_EXCEPTION(bool(*checkedTicket3));
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket3->DebugInfo());
- UNIT_ASSERT_NO_EXCEPTION(checkedTicket3->GetStatus());
- }
-
- Y_UNIT_TEST(TicketProtoTest) {
- ticket2::Ticket protobufTicket;
- UNIT_ASSERT(protobufTicket.ParseFromString(NUtils::Base64url2bin(USER_TICKET_PROTOBUF)));
- TTestUserTicketImpl userTicket(ETicketStatus::Ok, std::move(protobufTicket));
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, userTicket.GetStatus());
- UNIT_ASSERT_EQUAL(std::numeric_limits<time_t>::max(), userTicket.GetExpirationTime());
- UNIT_ASSERT_EQUAL(TUids({456, 123}), userTicket.GetUids());
- UNIT_ASSERT_EQUAL(456, userTicket.GetDefaultUid());
- UNIT_ASSERT_EQUAL(TScopes({"bb:sess1", "bb:sess2"}), userTicket.GetScopes());
- UNIT_ASSERT(userTicket.HasScope("bb:sess1"));
- UNIT_ASSERT(userTicket.HasScope("bb:sess2"));
- UNIT_ASSERT(!userTicket.HasScope("bb:sess3"));
- }
-
- Y_UNIT_TEST(ResetKeysTest) {
- TUserContext::TImpl context(EBlackboxEnv::Test, NUnittest::TVMKNIFE_PUBLIC_KEYS);
- context.ResetKeys(NUnittest::TVMKNIFE_PUBLIC_KEYS);
- auto checkedTicket = context.Check(VALID_USER_TICKET_1);
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, checkedTicket->GetStatus());
- }
-
- Y_UNIT_TEST(CreateTicketForTests) {
- TCheckedUserTicket t = NTvmAuth::NUnittest::CreateUserTicket(ETicketStatus::Ok, 42, {"qwerty", "omg"}, {43, 55, 47});
- UNIT_ASSERT_EQUAL(ETicketStatus::Ok, t.GetStatus());
- UNIT_ASSERT_EQUAL(42, t.GetDefaultUid());
- UNIT_ASSERT_EQUAL(TUids({42, 43, 47, 55}), t.GetUids());
- UNIT_ASSERT_EQUAL(TScopes({"omg", "qwerty"}), t.GetScopes());
- UNIT_ASSERT_VALUES_EQUAL("ticket_type=user;scope=omg;scope=qwerty;default_uid=42;uid=42;uid=43;uid=47;uid=55;env=Test;", t.DebugInfo());
- }
-
- Y_UNIT_TEST(CreateForTests) {
- TUids uids{456};
- TScopes scopes{"scope1", "scope2", "scope3"};
- TScopes scopesIn{"scope1", "scope2", "scope3", "scope1", ""};
- auto t = TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 456, scopesIn, {});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(456, t->GetDefaultUid());
- UNIT_ASSERT_VALUES_EQUAL(uids, t->GetUids());
- UNIT_ASSERT_VALUES_EQUAL(scopes, t->GetScopes());
-
- t = TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 456, scopesIn, {123, 456, 789});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(456, t->GetDefaultUid());
- uids = TUids{123, 456, 789};
- UNIT_ASSERT_VALUES_EQUAL(uids, t->GetUids());
- UNIT_ASSERT_VALUES_EQUAL(scopes, t->GetScopes());
-
- t = TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 456, scopesIn, {123, 789});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(456, t->GetDefaultUid());
- uids = TUids{123, 456, 789};
- UNIT_ASSERT_VALUES_EQUAL(uids, t->GetUids());
- UNIT_ASSERT_VALUES_EQUAL(scopes, t->GetScopes());
-
- t = TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 0, scopesIn, {123, 789});
- UNIT_ASSERT_VALUES_EQUAL(ETicketStatus::Ok, t->GetStatus());
- UNIT_ASSERT_VALUES_EQUAL(0, t->GetDefaultUid());
- uids = TUids{123, 789};
- UNIT_ASSERT_VALUES_EQUAL(uids, t->GetUids());
- UNIT_ASSERT_VALUES_EQUAL(scopes, t->GetScopes());
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 0, scopesIn, {}),
- yexception,
- "User ticket cannot contain empty uid list");
- UNIT_ASSERT_EXCEPTION_CONTAINS(TCheckedUserTicket::TImpl::CreateTicketForTests(ETicketStatus::Ok, 0, scopesIn, {0}),
- yexception,
- "User ticket cannot contain empty uid list");
- }
-}
-
-template <>
-void Out<NTvmAuth::TUids>(IOutputStream& o, const NTvmAuth::TUids& v) {
- for (const auto& uid : v) {
- o << uid << ",";
- }
-}
-
-template <>
-void Out<NTvmAuth::TScopes>(IOutputStream& o, const NTvmAuth::TScopes& v) {
- for (const auto& scope : v) {
- o << scope << ",";
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/utils_ut.cpp b/library/cpp/tvmauth/src/ut/utils_ut.cpp
deleted file mode 100644
index c9cb81c36f..0000000000
--- a/library/cpp/tvmauth/src/ut/utils_ut.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-#include <library/cpp/tvmauth/src/utils.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/maybe.h>
-
-Y_UNIT_TEST_SUITE(UtilsTestSuite) {
- static const TString VALID_SERVICE_TICKET_1 = "3:serv:CBAQ__________9_IhkI5QEQHBoIYmI6c2VzczEaCGJiOnNlc3My:WUPx1cTf05fjD1exB35T5j2DCHWH1YaLJon_a4rN-D7JfXHK1Ai4wM4uSfboHD9xmGQH7extqtlEk1tCTCGm5qbRVloJwWzCZBXo3zKX6i1oBYP_89WcjCNPVe1e8jwGdLsnu6PpxL5cn0xCksiStILH5UmDR6xfkJdnmMG94o8";
- static const TString EXPIRED_SERVICE_TICKET = "3:serv:CBAQACIZCOUBEBwaCGJiOnNlc3MxGghiYjpzZXNzMg:IwfMNJYEqStY_SixwqJnyHOMCPR7-3HHk4uylB2oVRkthtezq-OOA7QizDvx7VABLs_iTlXuD1r5IjufNei_EiV145eaa3HIg4xCdJXCojMexf2UYJz8mF2b0YzFAy6_KWagU7xo13CyKAqzJuQf5MJcSUf0ecY9hVh36cJ51aw";
- using namespace NTvmAuth;
-
- Y_UNIT_TEST(base64Test) {
- UNIT_ASSERT_VALUES_EQUAL("-hHx", NUtils::Bin2base64url("\xfa\x11\xf1"));
- UNIT_ASSERT_VALUES_EQUAL("-hHx_g", NUtils::Bin2base64url("\xfa\x11\xf1\xfe"));
- UNIT_ASSERT_VALUES_EQUAL("-hHx_v8", NUtils::Bin2base64url("\xfa\x11\xf1\xfe\xff"));
-
- UNIT_ASSERT_VALUES_EQUAL("", NUtils::Base64url2bin("hHx++"));
- UNIT_ASSERT_VALUES_EQUAL("", NUtils::Base64url2bin("&*^"));
- UNIT_ASSERT_VALUES_EQUAL("", NUtils::Base64url2bin(""));
- UNIT_ASSERT_VALUES_EQUAL("", NUtils::Bin2base64url(""));
-
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1", NUtils::Base64url2bin("-hHx"));
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1\xfe", NUtils::Base64url2bin("-hHx_g"));
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1\xfe", NUtils::Base64url2bin("-hHx_g="));
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1\xfe", NUtils::Base64url2bin("-hHx_g=="));
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1\xfe\xff", NUtils::Base64url2bin("-hHx_v8"));
- UNIT_ASSERT_VALUES_EQUAL("\xfa\x11\xf1\xfe\xff", NUtils::Base64url2bin("-hHx_v8="));
-
- UNIT_ASSERT_VALUES_EQUAL("SGVsbG8sIGV2ZXJ5Ym9keSE",
- NUtils::Bin2base64url(("Hello, everybody!")));
- UNIT_ASSERT_VALUES_EQUAL("Hello, everybody!",
- NUtils::Base64url2bin(("SGVsbG8sIGV2ZXJ5Ym9keSE")));
- UNIT_ASSERT_VALUES_EQUAL("VGhlIE1hZ2ljIFdvcmRzIGFyZSBTcXVlYW1pc2ggT3NzaWZyYWdl",
- NUtils::Bin2base64url(("The Magic Words are Squeamish Ossifrage")));
- UNIT_ASSERT_VALUES_EQUAL("The Magic Words are Squeamish Ossifrage",
- NUtils::Base64url2bin(("VGhlIE1hZ2ljIFdvcmRzIGFyZSBTcXVlYW1pc2ggT3NzaWZyYWdl")));
- }
-
- Y_UNIT_TEST(sign) {
- UNIT_ASSERT_VALUES_EQUAL("wkGfeuopf709ozPAeGcDMqtZXPzsWvuNJ1BL586dSug",
- NUtils::SignCgiParamsForTvm(NUtils::Base64url2bin("GRMJrKnj4fOVnvOqe-WyD1"),
- "1490000000",
- "13,19",
- "bb:sess,bb:sess2"));
-
- UNIT_ASSERT_VALUES_EQUAL("HANDYrA4ApQMQ5cfSWZk_InHWJffoXAa57P_X_B5s4M",
- NUtils::SignCgiParamsForTvm(NUtils::Base64url2bin("GRMJrKnj4fOasvOqe-WyD1"),
- "1490000000",
- "13,19",
- "bb:sess,bb:sess2"));
-
- UNIT_ASSERT_VALUES_EQUAL("T-M-3_qtjRM1dR_3hS1CRlHBTZRKK04doHXBJw-5VRk",
- NUtils::SignCgiParamsForTvm(NUtils::Base64url2bin("GRMJrKnj4fOasvOqe-WyD1"),
- "1490000001",
- "13,19",
- "bb:sess,bb:sess2"));
-
- UNIT_ASSERT_VALUES_EQUAL("gwB6M_9Jij50ZADmlDMnoyLc6AhQmtq6MClgGzO1PBE",
- NUtils::SignCgiParamsForTvm(NUtils::Base64url2bin("GRMJrKnj4fOasvOqe-WyD1"),
- "1490000001",
- "13,19",
- ""));
- }
-
- Y_UNIT_TEST(GetExpirationTime) {
- UNIT_ASSERT(!NTvmAuth::NInternal::TCanningKnife::GetExpirationTime("3:aadasdasdasdas"));
-
- UNIT_ASSERT(NTvmAuth::NInternal::TCanningKnife::GetExpirationTime(VALID_SERVICE_TICKET_1));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(std::numeric_limits<time_t>::max()),
- *NTvmAuth::NInternal::TCanningKnife::GetExpirationTime(VALID_SERVICE_TICKET_1));
-
- UNIT_ASSERT(NTvmAuth::NInternal::TCanningKnife::GetExpirationTime(EXPIRED_SERVICE_TICKET));
- UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(0),
- *NTvmAuth::NInternal::TCanningKnife::GetExpirationTime(EXPIRED_SERVICE_TICKET));
- }
-
- Y_UNIT_TEST(RemoveSignatureTest) {
- UNIT_ASSERT_VALUES_EQUAL("1:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("1:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("2:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("2:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("4:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("4:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3.serv.ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds",
- NUtils::RemoveTicketSignature("3.serv.ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs:asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("3:serv:",
- NUtils::RemoveTicketSignature("3:serv:ASDkljbjhsdbfLJHABFJHBslfbsfjs.asdxcvbxcvniueliuweklsvds"));
- UNIT_ASSERT_VALUES_EQUAL("asdxcbvfgdsgfasdfxczvdsgfxcdvbcbvf",
- NUtils::RemoveTicketSignature("asdxcbvfgdsgfasdfxczvdsgfxcdvbcbvf"));
- }
-}
diff --git a/library/cpp/tvmauth/src/ut/version_ut.cpp b/library/cpp/tvmauth/src/ut/version_ut.cpp
deleted file mode 100644
index eeb95d1cde..0000000000
--- a/library/cpp/tvmauth/src/ut/version_ut.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include <library/cpp/tvmauth/version.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <regex>
-
-using namespace NTvmAuth;
-
-Y_UNIT_TEST_SUITE(VersionTest) {
- Y_UNIT_TEST(base64Test) {
- const std::regex re(R"(^\d+\.\d+\.\d+$)");
-
- for (size_t idx = 0; idx < 2; ++idx) {
- TStringBuf ver = LibVersion();
- UNIT_ASSERT(std::regex_match(ver.begin(), ver.end(), re));
- }
- }
-}
diff --git a/library/cpp/tvmauth/src/utils.cpp b/library/cpp/tvmauth/src/utils.cpp
deleted file mode 100644
index d49efa28b5..0000000000
--- a/library/cpp/tvmauth/src/utils.cpp
+++ /dev/null
@@ -1,162 +0,0 @@
-#include "utils.h"
-
-#include "parser.h"
-
-#include <contrib/libs/openssl/include/openssl/evp.h>
-#include <contrib/libs/openssl/include/openssl/hmac.h>
-#include <contrib/libs/openssl/include/openssl/md5.h>
-#include <contrib/libs/openssl/include/openssl/sha.h>
-
-#include <util/generic/maybe.h>
-#include <util/generic/strbuf.h>
-
-#include <array>
-
-namespace {
- constexpr const unsigned char b64_encode[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
-
- constexpr std::array<unsigned char, 256> B64Init() {
- std::array<unsigned char, 256> buf{};
- for (auto& i : buf)
- i = 0xff;
-
- for (int i = 0; i < 64; ++i)
- buf[b64_encode[i]] = i;
-
- return buf;
- }
- constexpr std::array<unsigned char, 256> b64_decode = B64Init();
-}
-
-namespace NTvmAuth::NUtils {
- TString Bin2base64url(TStringBuf buf) {
- if (!buf) {
- return TString();
- }
-
- TString res;
- res.resize(((buf.size() + 2) / 3) << 2, 0);
-
- const unsigned char* pB = (const unsigned char*)buf.data();
- const unsigned char* pE = (const unsigned char*)buf.data() + buf.size();
- unsigned char* p = (unsigned char*)res.data();
- for (; pB + 2 < pE; pB += 3) {
- const unsigned char a = *pB;
- *p++ = b64_encode[(a >> 2) & 0x3F];
- const unsigned char b = *(pB + 1);
- *p++ = b64_encode[((a & 0x3) << 4) | ((b & 0xF0) >> 4)];
- const unsigned char c = *(pB + 2);
- *p++ = b64_encode[((b & 0xF) << 2) | ((c & 0xC0) >> 6)];
- *p++ = b64_encode[c & 0x3F];
- }
-
- if (pB < pE) {
- const unsigned char a = *pB;
- *p++ = b64_encode[(a >> 2) & 0x3F];
-
- if (pB == (pE - 1)) {
- *p++ = b64_encode[((a & 0x3) << 4)];
- } else {
- const unsigned char b = *(pB + 1);
- *p++ = b64_encode[((a & 0x3) << 4) |
- ((int)(b & 0xF0) >> 4)];
- *p++ = b64_encode[((b & 0xF) << 2)];
- }
- }
-
- res.resize(p - (unsigned char*)res.data());
- return res;
- }
-
- TString Base64url2bin(TStringBuf buf) {
- const unsigned char* bufin = (const unsigned char*)buf.data();
- if (!buf || b64_decode[*bufin] > 63) {
- return TString();
- }
- const unsigned char* bufend = (const unsigned char*)buf.data() + buf.size();
- while (++bufin < bufend && b64_decode[*bufin] < 64)
- ;
- int nprbytes = (bufin - (const unsigned char*)buf.data());
- int nbytesdecoded = ((nprbytes + 3) / 4) * 3;
-
- if (nprbytes < static_cast<int>(buf.size())) {
- int left = buf.size() - nprbytes;
- while (left--) {
- if (*(bufin++) != '=')
- return TString();
- }
- }
-
- TString res;
- res.resize(nbytesdecoded);
-
- unsigned char* bufout = (unsigned char*)res.data();
- bufin = (const unsigned char*)buf.data();
-
- while (nprbytes > 4) {
- unsigned char a = b64_decode[*bufin];
- unsigned char b = b64_decode[bufin[1]];
- *(bufout++) = (unsigned char)(a << 2 | b >> 4);
- unsigned char c = b64_decode[bufin[2]];
- *(bufout++) = (unsigned char)(b << 4 | c >> 2);
- unsigned char d = b64_decode[bufin[3]];
- *(bufout++) = (unsigned char)(c << 6 | d);
- bufin += 4;
- nprbytes -= 4;
- }
-
- if (nprbytes == 1) {
- return {}; // Impossible
- }
- if (nprbytes > 1) {
- *(bufout++) = (unsigned char)(b64_decode[*bufin] << 2 | b64_decode[bufin[1]] >> 4);
- }
- if (nprbytes > 2) {
- *(bufout++) = (unsigned char)(b64_decode[bufin[1]] << 4 | b64_decode[bufin[2]] >> 2);
- }
- if (nprbytes > 3) {
- *(bufout++) = (unsigned char)(b64_decode[bufin[2]] << 6 | b64_decode[bufin[3]]);
- }
-
- int diff = (4 - nprbytes) & 3;
- if (diff) {
- nbytesdecoded -= (4 - nprbytes) & 3;
- res.resize(nbytesdecoded);
- }
-
- return res;
- }
-
- TString SignCgiParamsForTvm(TStringBuf secret, TStringBuf ts, TStringBuf dstTvmId, TStringBuf scopes) {
- TString data;
- data.reserve(ts.size() + dstTvmId.size() + scopes.size() + 3);
- const char DELIM = '|';
- data.append(ts).push_back(DELIM);
- data.append(dstTvmId).push_back(DELIM);
- data.append(scopes).push_back(DELIM);
-
- TString value(EVP_MAX_MD_SIZE, 0);
- unsigned macLen = 0;
-
- if (!::HMAC(EVP_sha256(), secret.data(), secret.size(), (unsigned char*)data.data(), data.size(),
- (unsigned char*)value.data(), &macLen))
- {
- return {};
- }
-
- if (macLen != EVP_MAX_MD_SIZE) {
- value.resize(macLen);
- }
- return Bin2base64url(value);
- }
-}
-
-namespace NTvmAuth::NInternal {
- TMaybe<TInstant> TCanningKnife::GetExpirationTime(TStringBuf ticket) {
- const TParserTickets::TRes res = TParserTickets::ParseV3(ticket, {}, TParserTickets::ServiceFlag());
-
- return res.Status == ETicketStatus::MissingKey || res.Status == ETicketStatus::Expired
- ? TInstant::Seconds(res.Ticket.expirationtime())
- : TMaybe<TInstant>();
- }
-}
diff --git a/library/cpp/tvmauth/src/utils.h b/library/cpp/tvmauth/src/utils.h
deleted file mode 100644
index e5847ac89f..0000000000
--- a/library/cpp/tvmauth/src/utils.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#pragma once
-
-#include <library/cpp/tvmauth/checked_service_ticket.h>
-#include <library/cpp/tvmauth/checked_user_ticket.h>
-#include <library/cpp/tvmauth/ticket_status.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/fwd.h>
-
-namespace NTvmAuth::NUtils {
- TString Bin2base64url(TStringBuf buf);
- TString Base64url2bin(TStringBuf buf);
-
- TString SignCgiParamsForTvm(TStringBuf secret, TStringBuf ts, TStringBuf dstTvmId, TStringBuf scopes);
-}
-
-namespace NTvmAuth::NInternal {
- class TCanningKnife {
- public:
- static TCheckedServiceTicket::TImpl* GetS(TCheckedServiceTicket& t) {
- return t.Impl_.Release();
- }
-
- static TCheckedUserTicket::TImpl* GetU(TCheckedUserTicket& t) {
- return t.Impl_.Release();
- }
-
- static TMaybe<TInstant> GetExpirationTime(TStringBuf ticket);
- };
-}
diff --git a/library/cpp/tvmauth/src/version b/library/cpp/tvmauth/src/version
deleted file mode 100644
index 6cb9d3dd0d..0000000000
--- a/library/cpp/tvmauth/src/version
+++ /dev/null
@@ -1 +0,0 @@
-3.4.3
diff --git a/library/cpp/tvmauth/src/version.cpp b/library/cpp/tvmauth/src/version.cpp
deleted file mode 100644
index 6b389213d0..0000000000
--- a/library/cpp/tvmauth/src/version.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#include <library/cpp/resource/resource.h>
-
-#include <util/string/strip.h>
-
-namespace {
- class TBuiltinVersion {
- public:
- TBuiltinVersion() {
- Version_ = NResource::Find("/builtin/version");
- StripInPlace(Version_);
- }
-
- TStringBuf Get() const {
- return Version_;
- }
-
- private:
- TString Version_;
- };
-}
-
-namespace NTvmAuth {
- TStringBuf LibVersion() {
- return Singleton<TBuiltinVersion>()->Get();
- }
-}
diff --git a/library/cpp/tvmauth/ticket_status.h b/library/cpp/tvmauth/ticket_status.h
deleted file mode 100644
index 532d4de56e..0000000000
--- a/library/cpp/tvmauth/ticket_status.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#pragma once
-
-#include <util/generic/strbuf.h>
-
-namespace NTvmAuth {
- /*!
- * Status mean result of ticket check
- */
- enum class ETicketStatus {
- Ok,
- Expired,
- InvalidBlackboxEnv,
- InvalidDst,
- InvalidTicketType,
- Malformed,
- MissingKey,
- SignBroken,
- UnsupportedVersion,
- NoRoles,
- };
-
- TStringBuf StatusToString(ETicketStatus st);
-}
diff --git a/library/cpp/tvmauth/type.h b/library/cpp/tvmauth/type.h
deleted file mode 100644
index 7f4ce2b700..0000000000
--- a/library/cpp/tvmauth/type.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <library/cpp/containers/stack_vector/stack_vec.h>
-
-namespace NTvmAuth {
- using TScopes = TSmallVec<TStringBuf>;
- using TTvmId = ui32;
- using TUid = ui64;
- using TUids = TSmallVec<TUid>;
- using TAlias = TString;
-}
diff --git a/library/cpp/tvmauth/unittest.h b/library/cpp/tvmauth/unittest.h
deleted file mode 100644
index efa651befa..0000000000
--- a/library/cpp/tvmauth/unittest.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include "checked_service_ticket.h"
-#include "checked_user_ticket.h"
-
-#include <util/generic/maybe.h>
-
-namespace NTvmAuth::NUnittest {
- static const TString TVMKNIFE_PUBLIC_KEYS = "1:CpgCCpMCCAEQABqIAjCCAQQCggEAcLEXeH67FQESFUn4_7wnX7wN0PUrBoUsm3QQ4W5vC-qz6sXaEjSwnTV8w1o-z6X9KPLlhzMQvuS38NCNfK4uvJ4Zvfp3YsXJ25-rYtbnrYJHNvHohD-kPCCw_yZpMp21JdWigzQGuV7CtrxUhF-NNrsnUaJrE5-OpEWNt4X6nCItKIYeVcSK6XJUbEWbrNCRbvkSc4ak2ymFeMuHYJVjxh4eQbk7_ZPzodP0WvF6eUYrYeb42imVEOR8ofVLQWE5DVnb1z_TqZm4i1XkS7jMwZuBxBRw8DGdYei0lT_sAf7KST2jC0590NySB3vsBgWEVs1OdUUWA6r-Dvx9dsOQtSCVkQYQAAqZAgqUAggCEAAaiQIwggEFAoIBAQDhEBM5-6YsPWfogKtbluJoCX1WV2KdzOaQ0-OlRbBzeCzw-eQKu12c8WakHBbeCMd1I1TU64SDkDorWjXGIa_2xT6N3zzNAE50roTbPCcmeQrps26woTYfYIuqDdoxYKZNr0lvNLLW47vBr7EKqo1S4KSj7aXK_XYeEvUgIgf3nVIcNrio7VTnFmGGVQCepaL1Hi1gN4yIXjVZ06PBPZ-DxSRu6xOGbFrfKMJeMPs7KOyE-26Q3xOXdTIa1X-zYIucTd_bxUCL4BVbwW2AvbbFsaG7ISmVdGu0XUTmhXs1KrEfUVLRJhE4Dx99hAZXm1_HlYMUeJcMQ_oHOhV94ENFIJaRBhACCpYBCpEBCAMQABqGATCBgwKBgF9t2YJGAJkRRFq6fWhi3m1TFW1UOE0f6ZrfYhHAkpqGlKlh0QVfeTNPpeJhi75xXzCe6oReRUm-0DbqDNhTShC7uGUv1INYnRBQWH6E-5Fc5XrbDFSuGQw2EYjNfHy_HefHJXxQKAqPvxBDKMKkHgV58WtM6rC8jRi9sdX_ig2NIJeRBhABCpYBCpEBCAQQABqGATCBgwKBgGB4d6eLGUBv-Q6EPLehC4S-yuE2HB-_rJ7WkeYwyp-xIPolPrd-PQme2utHB4ZgpXHIu_OFksDe_0bPgZniNRSVRbl7W49DgS5Ya3kMfrYB4DnF5Fta5tn1oV6EwxYD4JONpFTenOJALPGTPawxXEfon_peiHOSBuQMu3_Vn-l1IJiRBhADCpcBCpIBCAUQABqHATCBhAKBgQCTJMKIfmfeZpaI7Q9rnsc29gdWawK7TnpVKRHws1iY7EUlYROeVcMdAwEqVM6f8BVCKLGgzQ7Gar_uuxfUGKwqEQzoppDraw4F75J464-7D5f6_oJQuGIBHZxqbMONtLjBCXRUhQW5szBLmTQ_R3qaJb5vf-h0APZfkYhq1cTttSCZkQYQBAqWAQqRAQgLEAAahgEwgYMCgYBvvGVH_M2H8qxxv94yaDYUTWbRnJ1uiIYc59KIQlfFimMPhSS7x2tqUa2-hI55JiII0Xym6GNkwLhyc1xtWChpVuIdSnbvttbrt4weDMLHqTwNOF6qAsVKGKT1Yh8yf-qb-DSmicgvFc74mBQm_6gAY1iQsf33YX8578ClhKBWHSCVkQYQAAqXAQqSAQgMEAAahwEwgYQCgYEAkuzFcd5TJu7lYWYe2hQLFfUWIIj91BvQQLa_Thln4YtGCO8gG1KJqJm-YlmJOWQG0B7H_5RVhxUxV9KpmFnsDVkzUFKOsCBaYGXc12xPVioawUlAwp5qp3QQtZyx_se97YIoLzuLr46UkLcLnkIrp-Jo46QzYi_QHq45WTm8MQ0glpEGEAIKlwEKkgEIDRAAGocBMIGEAoGBAIUzbxOknXf_rNt17_ir8JlWvrtnCWsQd1MAnl5mgArvavDtKeBYHzi5_Ak7DHlLzuA6YE8W175FxLFKpN2hkz-l-M7ltUSd8N1BvJRhK4t6WffWfC_1wPyoAbeSN2Yb1jygtZJQ8wGoXHcJQUXiMit3eFNyylwsJFj1gzAR4JCdIJeRBhABCpYBCpEBCA4QABqGATCBgwKBgFMcbEpl9ukVR6AO_R6sMyiU11I8b8MBSUCEC15iKsrVO8v_m47_TRRjWPYtQ9eZ7o1ocNJHaGUU7qqInFqtFaVnIceP6NmCsXhjs3MLrWPS8IRAy4Zf4FKmGOx3N9O2vemjUygZ9vUiSkULdVrecinRaT8JQ5RG4bUMY04XGIwFIJiRBhADCpYBCpEBCA8QABqGATCBgwKBgGpCkW-NR3li8GlRvqpq2YZGSIgm_PTyDI2Zwfw69grsBmPpVFW48Vw7xoMN35zcrojEpialB_uQzlpLYOvsMl634CRIuj-n1QE3-gaZTTTE8mg-AR4mcxnTKThPnRQpbuOlYAnriwiasWiQEMbGjq_HmWioYYxFo9USlklQn4-9IJmRBhAEEpUBCpIBCAYQABqHATCBhAKBgQCoZkFGm9oLTqjeXZAq6j5S6i7K20V0lNdBBLqfmFBIRuTkYxhs4vUYnWjZrKRAd5bp6_py0csmFmpl_5Yh0b-2pdo_E5PNP7LGRzKyKSiFddyykKKzVOazH8YYldDAfE8Z5HoS9e48an5JsPg0jr-TPu34DnJq3yv2a6dqiKL9zSCakQYSlQEKkgEIEBAAGocBMIGEAoGBALhrihbf3EpjDQS2sCQHazoFgN0nBbE9eesnnFTfzQELXb2gnJU9enmV_aDqaHKjgtLIPpCgn40lHrn5k6mvH5OdedyI6cCzE-N-GFp3nAq0NDJyMe0fhtIRD__CbT0ulcvkeow65ubXWfw6dBC2gR_34rdMe_L_TGRLMWjDULbNIJqRBg";
-
- TCheckedServiceTicket CreateServiceTicket(ETicketStatus status,
- TTvmId src,
- TMaybe<TUid> issuerUid = TMaybe<TUid>());
-
- TCheckedUserTicket CreateUserTicket(ETicketStatus status,
- TUid defaultUid,
- const TScopes& scopes,
- const TUids& uids = TUids(),
- EBlackboxEnv env = EBlackboxEnv::Test);
-}
diff --git a/library/cpp/tvmauth/utils.cpp b/library/cpp/tvmauth/utils.cpp
deleted file mode 100644
index a06cd6f5ba..0000000000
--- a/library/cpp/tvmauth/utils.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include "utils.h"
-
-namespace NTvmAuth::NUtils {
- TStringBuf RemoveTicketSignature(TStringBuf ticketBody) {
- if (ticketBody.size() < 2 ||
- ticketBody[0] != '3' ||
- ticketBody[1] != ':') {
- return ticketBody;
- }
-
- size_t pos = ticketBody.rfind(':');
- if (pos == TStringBuf::npos) { // impossible
- return ticketBody;
- }
-
- return ticketBody.substr(0, pos + 1);
- }
-}
diff --git a/library/cpp/tvmauth/utils.h b/library/cpp/tvmauth/utils.h
deleted file mode 100644
index ad8950cab5..0000000000
--- a/library/cpp/tvmauth/utils.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#pragma once
-
-#include <util/generic/strbuf.h>
-
-namespace NTvmAuth::NUtils {
- /*!
- * Remove signature from ticket string - rest part can be parsed later with `tvmknife parse_ticket ...`
- * @param ticketBody Raw ticket body
- * @return safe for logging part of ticket
- */
- TStringBuf RemoveTicketSignature(TStringBuf ticketBody);
-}
diff --git a/library/cpp/tvmauth/version.h b/library/cpp/tvmauth/version.h
deleted file mode 100644
index 48ec279829..0000000000
--- a/library/cpp/tvmauth/version.h
+++ /dev/null
@@ -1,7 +0,0 @@
-#pragma once
-
-#include <util/generic/strbuf.h>
-
-namespace NTvmAuth {
- TStringBuf LibVersion();
-}
diff --git a/ydb/core/client/server/pq_metacache_v2_ut.cpp b/ydb/core/client/server/pq_metacache_v2_ut.cpp
deleted file mode 100644
index bb26707fa8..0000000000
--- a/ydb/core/client/server/pq_metacache_v2_ut.cpp
+++ /dev/null
@@ -1,355 +0,0 @@
-#include "msgbus_server_pq_metacache.h"
-
-#include <ydb/core/testlib/basics/runtime.h>
-#include <ydb/core/testlib/test_client.h>
-#include <ydb/core/testlib/test_pq_client.h>
-#include <ydb/core/scheme_types/scheme_type_registry.h>
-#include <ydb/core/tx/scheme_board/cache.h>
-#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h>
-
-namespace NKikimr::NMsgBusProxy {
-using namespace NPqMetaCacheV2;
-using namespace Tests;
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NKikimr::NSchemeCache;
-namespace NTests {
-
-class TPqMetaCacheV2Test: public TTestBase {
- void SetUp() override {
- TTestBase::SetUp();
- NKikimr::NPersQueueTests::TServerSettings settings = NKikimr::NPersQueueTests::PQSettings(0);
- settings.SetGrpcPort(GrpcServerPort);
- settings.SetDomainName("Root");
- settings.SetDomain(0);
- settings.SetUseRealThreads(true);
- Server = MakeHolder<::NPersQueue::TTestServer>(settings);
- GrpcServerPort = Server->GrpcPort;
- auto* runtime = Server->CleverServer->GetRuntime();
- Server->EnableLogs({NKikimrServices::PQ_METACACHE, NKikimrServices::PQ_WRITE_PROXY});
- Server->EnableLogs({NKikimrServices::PERSQUEUE}, NActors::NLog::PRI_INFO);
- //runtime->SetLogPriority(NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, NActors::NLog::PRI_EMERG);
-
-
- NYdb::TDriverConfig driverCfg;
- driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << GrpcServerPort);
- YdbDriver = std::make_shared<NYdb::TDriver>(driverCfg);
- TableClient.Reset(new NYdb::NTable::TTableClient(*YdbDriver));
- PQClient = std::make_shared<NYdb::NPersQueue::TPersQueueClient>(*YdbDriver);
-
- auto tableDesc = TTableBuilder()
- .AddNullableColumn("path", EPrimitiveType::Utf8)
- .AddNullableColumn("dc", EPrimitiveType::Utf8)
- .SetPrimaryKeyColumns({"path", "dc"})
- .Build();
-
- auto session = CheckYdbResult(std::move(TableClient->CreateSession())).GetSession();
- CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Topics", std::move(tableDesc)));
-
- tableDesc = TTableBuilder()
- .AddNullableColumn("name", EPrimitiveType::String)
- .AddNullableColumn("version", EPrimitiveType::Int64)
- .SetPrimaryKeyColumns({"name"})
- .Build();
- CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Versions", std::move(tableDesc)));
-
- EdgeActorId = runtime->AllocateEdgeActor();
- auto config = MakeIntrusive<TSchemeCacheConfig>();
- config->Counters = new NMonitoring::TDynamicCounters;
- config->Roots.emplace_back(settings.Domain, 8716544, "Root");
- auto& domainsInfo = settings.AppConfig.GetDomainsConfig();
- for (const auto& domain : domainsInfo.GetDomain()) {
- config->Roots.emplace_back(domain.GetDomainId(), domain.GetSchemeRoot(), domain.GetName());
- }
- SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get()));
- MetaCacheId = runtime->Register(
- NPqMetaCacheV2::CreatePQMetaCache(config->Counters, TDuration::MilliSeconds(50))
- );
- runtime->EnableScheduleForActor(SchemeCacheId, true);
- runtime->EnableScheduleForActor(MetaCacheId, true);
- }
-
- struct TTopicInfo {
- TString Path;
- TString Cluster;
- bool DoCreate;
- };
-
- void AddTopics(const TVector<TTopicInfo>& topics, bool shiftVersion = true) {
- auto session = TableClient->GetSession().GetValueSync().GetSession();
- auto topicsPrepared = CheckYdbResult(session.PrepareDataQuery(UpsertTopicQuery)).GetQuery();
- TVector<TAsyncStatus> createTopicStatus;
- auto tx = session.BeginTransaction().GetValueSync().GetTransaction();
- auto txControl = TTxControl::Tx(tx);
- for (const auto& topic : topics) {
- auto fullPath = "/Root/PQ/" + ::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster);
- if (topic.DoCreate) {
- createTopicStatus.push_back(PQClient->CreateTopic(fullPath));
- }
- auto builder = topicsPrepared.GetParamsBuilder();
- {
- auto &param = builder.AddParam("$Path");
- param.Utf8(topic.Path);
- param.Build();
- }
- {
- auto &param = builder.AddParam("$Cluster");
- param.Utf8(topic.Cluster);
- param.Build();
- }
- CheckYdbResult(topicsPrepared.Execute(txControl, builder.Build()));
- }
- if (shiftVersion) {
- auto versionPrepared = CheckYdbResult(session.PrepareDataQuery(UpdateVersionQuery)).GetQuery();
- auto builder = versionPrepared.GetParamsBuilder();
- {
- auto &param = builder.AddParam("$Version");
- param.Int64(++Version);
- param.Build();
- }
- CheckYdbResult(versionPrepared.Execute(txControl, builder.Build()));
- Cerr << "TOPIC VERSION SHIFTED TO " << Version << "\n";
- }
- CheckYdbResult(tx.Commit());
- for (auto& status : createTopicStatus) {
- CheckYdbResult(std::move(status));
- }
- }
-
- template<class TEvType>
- THolder<TEvType> DoMetaCacheRequest(const TVector<TTopicInfo>& topicList = {}) {
- auto factory = std::make_shared<::NPersQueue::TTopicNamesConverterFactory>(false, "/Root/PQ", "dc1");
- IEventBase* ev;
- if (topicList.empty()) {
- ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest();
- } else {
- TVector<::NPersQueue::TDiscoveryConverterPtr> topicNames;
- for (const auto &topic : topicList) {
- topicNames.emplace_back(factory->MakeDiscoveryConverter(topic.Path, false, topic.Cluster));
- }
- ev = new TEvPqNewMetaCache::TEvDescribeTopicsRequest(topicNames);
- }
- auto handle = new IEventHandle(MetaCacheId, EdgeActorId, ev);
- Server->CleverServer->GetRuntime()->Send(handle);
- auto response = Server->CleverServer->GetRuntime()->GrabEdgeEvent<TEvType>();
- return std::move(response);
- }
-
- THolder<TEvPqNewMetaCache::TEvDescribeTopicsResponse> DescribeTopics(const TVector<TTopicInfo>& topicList) {
- return DoMetaCacheRequest<TEvPqNewMetaCache::TEvDescribeTopicsResponse>(topicList);
- }
-
- THolder<TEvPqNewMetaCache::TEvDescribeAllTopicsResponse> DescribeAllTopics() {
- return DoMetaCacheRequest<TEvPqNewMetaCache::TEvDescribeAllTopicsResponse>();
- }
-
- void CheckTopicInfo(const TVector<TTopicInfo>& expected, std::shared_ptr<NSchemeCache::TSchemeCacheNavigate>& result) {
- ui64 i = 0;
- Cerr << "=== Got cache navigate response: \n" << result->ToString(NKikimr::NScheme::TTypeRegistry()) << Endl;
- Cerr << "=== Expect to have " << expected.size() << " records, got: " << result->ResultSet.size() << " records" << Endl;
- UNIT_ASSERT_VALUES_EQUAL_C(expected.size(), result->ResultSet.size(), "Result size mismatch");
- for (const auto& topic : expected) {
- Cerr << "===Check topic: " << topic.Path << ":" << topic.Cluster << Endl;
- auto& response = result->ResultSet[i++];
- if (topic.DoCreate) {
- UNIT_ASSERT(response.Status == TSchemeCacheNavigate::EStatus::Ok);
- UNIT_ASSERT(response.PQGroupInfo);
- UNIT_ASSERT(response.PQGroupInfo->Kind == NSchemeCache::TSchemeCacheNavigate::KindTopic);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(response.Status, TSchemeCacheNavigate::EStatus::PathErrorUnknown);
- }
- }
- }
-
- void WaitForVersion(ui64 version, TDuration timeout = TDuration::Seconds(5)) {
- ui64 currentVersion = 0;
- auto endTime = TInstant::Now() + timeout;
- auto* runtime = Server->CleverServer->GetRuntime();
- while (endTime > TInstant::Now()) {
- auto handle = new IEventHandle(MetaCacheId, EdgeActorId, new TEvPqNewMetaCache::TEvGetVersionRequest());
- runtime->Send(handle);
- auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvGetVersionResponse>();
- currentVersion = response->TopicsVersion;
- if (currentVersion >= version) {
- Cerr << "=== Got current topics version: " << currentVersion << Endl;
- return;
- }
- Sleep(TDuration::MilliSeconds(100));
- }
- UNIT_FAIL("Wait for topics version timed out");
- }
-
- TActorId MakeEdgeTargetedMetaCache() {
- auto anotherMetaCacheId = Server->CleverServer->GetRuntime()->Register(
- NPqMetaCacheV2::CreatePQMetaCache(EdgeActorId)
-
- );
- return anotherMetaCacheId;
- }
-
- void TestDescribeTopics() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "man", true},
- {"topic2", "man", true},
- {"topic3", "man", false}
- });
- AddTopics(topics);
- auto ev = DescribeTopics(topics);
- CheckTopicInfo(topics, ev->Result);
- UNIT_ASSERT(ev);
- }
-
- void TestDescribeManyTopics() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "man", true},
- {"topic2", "man", false},
- {"topic3", "man", false}
- });
-
- AddTopics(topics);
-
- Cerr << "===Wait base response\n";
- auto baseResponse = DescribeTopics(topics);
- Cerr << "===Got base response\n";
- auto baseEntry = baseResponse->Result->ResultSet[0];
-
- TVector<TTopicInfo> lotOfTopics;
- for (auto i = 0u; i < 10000u; i++) {
- lotOfTopics.emplace_back(TTopicInfo{TString("topic0") + ToString(i), "man", false});
- }
- AddTopics(lotOfTopics);
- ui64 totalTopics = topics.size() + lotOfTopics.size();
- auto secondMetaCache = MakeEdgeTargetedMetaCache();
- Cerr << "===Registered secondary meta cache: " << secondMetaCache.ToString() << Endl;
- auto* runtime = Server->CleverServer->GetRuntime();
-
- auto sendRequest = [&]() {
- auto* ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest();
- auto handle = new IEventHandle(secondMetaCache, EdgeActorId, ev);
- runtime->Send(handle);
- };
- Cerr << "===Send first request\n";
- sendRequest();
-
- auto scRequest = runtime->GrabEdgeEvent<TEvTxProxySchemeCache::TEvNavigateKeySet>();
- Cerr << "===Got SC request\n";
- auto& entries = scRequest->Request->ResultSet;
- UNIT_ASSERT_VALUES_EQUAL(entries.size(), totalTopics);
- for (auto i = 0u; i < entries.size(); i++) {
- auto initialPath = entries[i].Path;
- entries[i] = baseEntry;
- entries[i].Path = initialPath;
- }
- auto* scResponse = new TEvTxProxySchemeCache::TEvNavigateKeySetResult(scRequest->Request);
-
- auto handle = new IEventHandle(secondMetaCache, EdgeActorId, scResponse);
- Cerr << "===Send fake SC response\n";
- runtime->Send(handle);
-
- Cerr << "===Wait secondary metacache response\n";
- auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeAllTopicsResponse>();
- Cerr << "===Got secondary metacache response\n";
-
- // Preparation done, now can send many requests
- Cerr << "===Started pushing requests at: " << TInstant::Now() << Endl;
- for (auto i = 0u; i < 1000; i++) {
- sendRequest();
- }
- Cerr << "===Done pushing requests at: " << TInstant::Now() << Endl;
- for (auto i = 0u; i < 1000; i++) {
- response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeAllTopicsResponse>();
- if (!i) {
- Cerr << "===Done gathering first response at: " << TInstant::Now() << Endl;
- }
- }
- Cerr << "===Done gathering responses at: " << TInstant::Now() << Endl;
- }
-
- void TestDescribeAllTopics() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "dc1", true},
- {"topic2", "dc1", true},
- {"topic3", "dc1", false}
- });
- AddTopics(topics);
- auto ev = DescribeAllTopics();
- CheckTopicInfo(topics, ev->Result);
- UNIT_ASSERT(ev);
- }
-
- void TestTopicsUpdate() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "dc1", true},
- {"topic2", "dc1", true},
- {"topic3", "dc1", false}
- });
- AddTopics(topics);
- CheckTopicInfo(topics, DescribeTopics(topics)->Result);
- WaitForVersion(Version);
-
- TTopicInfo topic{"topic1", "dc2", false};
- Server->AnnoyingClient->CreateTopic(NPersQueueTests::TRequestCreatePQ("rt3.dc2--topic1", 1), false);
- AddTopics({topic}, false);
-
- CheckTopicInfo(topics, DescribeAllTopics()->Result);
- topic.DoCreate = true;
-
- AddTopics({}, true);
- WaitForVersion(Version);
- topics.insert(topics.end() - 1, topic);
- CheckTopicInfo(topics, DescribeAllTopics()->Result);
- }
-
-
- template<class T>
- T CheckYdbResult(NThreading::TFuture<T>&& asyncResult) {
- auto res = asyncResult.GetValueSync();
- UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
- return res;
- }
-
- TString GetUpsertTopicQuery(const TTopicInfo& info) {
- return TStringBuilder()
- << "--!syntax_v1\n"
- << "UPSERT INTO `/Root/PQ/Config/V2/Topics` "
- << "(path, dc) VALUES (" << info.Path << ", " << info.Cluster << ");";
- }
-
- UNIT_TEST_SUITE(TPqMetaCacheV2Test)
- UNIT_TEST(TestDescribeTopics)
- UNIT_TEST(TestDescribeAllTopics)
- UNIT_TEST(TestTopicsUpdate)
- UNIT_TEST_SUITE_END();
-private:
- ui16 GrpcServerPort;
- THolder<::NPersQueue::TTestServer> Server;
-
- TActorId EdgeActorId;
- TActorId SchemeCacheId;
- TActorId MetaCacheId;
- ui64 Version = 1;
-
- std::shared_ptr<NYdb::TDriver> YdbDriver;
- TAtomicSharedPtr<NYdb::NTable::TTableClient> TableClient;
- std::shared_ptr<NYdb::NPersQueue::TPersQueueClient> PQClient;
- TString UpsertTopicQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; "
- << "UPSERT INTO `/Root/PQ/Config/V2/Topics`"
- << "(path, dc) VALUES ($Path, $Cluster);";
- TString UpdateVersionQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Version as Int64; "
- << "UPSERT INTO `/Root/PQ/Config/V2/Versions`"
- << "(name, version) VALUES ('Topics', $Version);";
-
-
-};
-UNIT_TEST_SUITE_REGISTRATION(TPqMetaCacheV2Test);
-
-}// namespace NTests
-} // namespace NKikimr::NMsgBusProxy
diff --git a/ydb/core/client/server/ut/CMakeLists.darwin.txt b/ydb/core/client/server/ut/CMakeLists.darwin.txt
index c3c882febf..094ca6798c 100644
--- a/ydb/core/client/server/ut/CMakeLists.darwin.txt
+++ b/ydb/core/client/server/ut/CMakeLists.darwin.txt
@@ -25,7 +25,6 @@ target_link_libraries(ydb-core-client-server-ut PUBLIC
ydb-core-tablet_flat
ydb-core-testlib
core-testlib-actors
- cpp-v2-ut_utils
)
target_link_options(ydb-core-client-server-ut PRIVATE
-Wl,-no_deduplicate
@@ -36,7 +35,6 @@ target_link_options(ydb-core-client-server-ut PRIVATE
CoreFoundation
)
target_sources(ydb-core-client-server-ut PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/core/client/server/pq_metacache_v2_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
)
add_test(
diff --git a/ydb/core/client/server/ut/CMakeLists.linux.txt b/ydb/core/client/server/ut/CMakeLists.linux.txt
index 39c1d994e2..1602daf50a 100644
--- a/ydb/core/client/server/ut/CMakeLists.linux.txt
+++ b/ydb/core/client/server/ut/CMakeLists.linux.txt
@@ -26,7 +26,6 @@ target_link_libraries(ydb-core-client-server-ut PUBLIC
ydb-core-tablet_flat
ydb-core-testlib
core-testlib-actors
- cpp-v2-ut_utils
)
target_link_options(ydb-core-client-server-ut PRIVATE
-ldl
@@ -39,7 +38,6 @@ target_link_options(ydb-core-client-server-ut PRIVATE
-ldl
)
target_sources(ydb-core-client-server-ut PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/core/client/server/pq_metacache_v2_ut.cpp
${CMAKE_SOURCE_DIR}/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
)
add_test(