diff options
| author | aozeritsky <aozeritsky@yandex-team.ru> | 2022-02-10 16:46:40 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:40 +0300 |
| commit | b036a557f285146e5e35d4213e29a094ab907bcf (patch) | |
| tree | 49e222ea1c5804306084bb3ae065bb702625360f | |
| parent | 4a6816dea1bcaee46ce29a51a5fd7d3495012858 (diff) | |
| download | ydb-b036a557f285146e5e35d4213e29a094ab907bcf.tar.gz | |
Restoring authorship annotation for <aozeritsky@yandex-team.ru>. Commit 2 of 2.
346 files changed, 19727 insertions, 19727 deletions
diff --git a/contrib/libs/libevent/evutil.c b/contrib/libs/libevent/evutil.c index 159dd08609..9976e79beb 100644 --- a/contrib/libs/libevent/evutil.c +++ b/contrib/libs/libevent/evutil.c @@ -1250,11 +1250,11 @@ evutil_adjust_hints_for_addrconfig_(struct evutil_addrinfo *hints) #ifdef USE_NATIVE_GETADDRINFO static int need_numeric_port_hack_=0; static int need_socktype_protocol_hack_=0; -#ifdef WIN32 -static int tested_for_getaddrinfo_hacks=1; -#else +#ifdef WIN32 +static int tested_for_getaddrinfo_hacks=1; +#else static int tested_for_getaddrinfo_hacks=0; -#endif +#endif /* Some older BSDs (like OpenBSD up to 4.6) used to believe that giving a numeric port without giving an ai_socktype was verboten. We test for this so we can apply an appropriate workaround. If it diff --git a/contrib/libs/libevent/http-internal.h b/contrib/libs/libevent/http-internal.h index 95d70525ec..85760eafc2 100644 --- a/contrib/libs/libevent/http-internal.h +++ b/contrib/libs/libevent/http-internal.h @@ -13,7 +13,7 @@ #include "event2/event_struct.h" #include "util-internal.h" #include "defer-internal.h" -#include "event2/http.h" +#include "event2/http.h" #define HTTP_CONNECT_TIMEOUT 45 #define HTTP_WRITE_TIMEOUT 50 @@ -56,8 +56,8 @@ struct evhttp_connection { evutil_socket_t fd; struct bufferevent *bufev; - bev_factory_cb bufcb; - void *bufcb_arg; + bev_factory_cb bufcb; + void *bufcb_arg; struct event retry_ev; /* for retrying connects */ @@ -112,7 +112,7 @@ struct evhttp_cb { TAILQ_ENTRY(evhttp_cb) next; char *what; - int chunked; + int chunked; void (*cb)(struct evhttp_request *req, void *); void *cbarg; @@ -169,8 +169,8 @@ struct evhttp { don't match. */ void (*gencb)(struct evhttp_request *req, void *); void *gencbarg; - struct bufferevent* (*bevcb)(struct event_base *, void *); - void *bevcbarg; + struct bufferevent* (*bevcb)(struct event_base *, void *); + void *bevcbarg; struct event_base *base; }; diff --git a/contrib/libs/libevent/http.c b/contrib/libs/libevent/http.c index e3ce49f0ba..b750d491c9 100644 --- a/contrib/libs/libevent/http.c +++ b/contrib/libs/libevent/http.c @@ -200,9 +200,9 @@ static void evhttp_write_buffer(struct evhttp_connection *, void (*)(struct evhttp_connection *, void *), void *); static void evhttp_make_header(struct evhttp_connection *, struct evhttp_request *); -static struct evhttp_cb * -evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req, int chunked); - +static struct evhttp_cb * +evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req, int chunked); + /* callbacks for bufferevent */ static void evhttp_read_cb(struct bufferevent *, void *); static void evhttp_write_cb(struct bufferevent *, void *); @@ -983,7 +983,7 @@ evhttp_handle_chunked_read(struct evhttp_request *req, struct evbuffer *buf) req->ntoread = -1; if (req->chunk_cb != NULL) { req->flags |= EVHTTP_REQ_DEFER_FREE; - (*req->chunk_cb)(req, req->chunk_cb_arg); + (*req->chunk_cb)(req, req->chunk_cb_arg); evbuffer_drain(req->input_buffer, evbuffer_get_length(req->input_buffer)); req->flags &= ~EVHTTP_REQ_DEFER_FREE; @@ -993,12 +993,12 @@ evhttp_handle_chunked_read(struct evhttp_request *req, struct evbuffer *buf) } } - if (req->chunk_cb != NULL) { - req->flags |= EVHTTP_REQ_PROCESS_CHUNKS_END; - (*req->chunk_cb)(req, req->chunk_cb_arg); - req->flags &= ~EVHTTP_REQ_PROCESS_CHUNKS_END; - } - + if (req->chunk_cb != NULL) { + req->flags |= EVHTTP_REQ_PROCESS_CHUNKS_END; + (*req->chunk_cb)(req, req->chunk_cb_arg); + req->flags &= ~EVHTTP_REQ_PROCESS_CHUNKS_END; + } + return (MORE_DATA_EXPECTED); } @@ -1112,7 +1112,7 @@ evhttp_read_body(struct evhttp_connection *evcon, struct evhttp_request *req) if (evbuffer_get_length(req->input_buffer) > 0 && req->chunk_cb != NULL) { req->flags |= EVHTTP_REQ_DEFER_FREE; - (*req->chunk_cb)(req, req->chunk_cb_arg); + (*req->chunk_cb)(req, req->chunk_cb_arg); req->flags &= ~EVHTTP_REQ_DEFER_FREE; evbuffer_drain(req->input_buffer, evbuffer_get_length(req->input_buffer)); @@ -1364,23 +1364,23 @@ evhttp_connection_reset_(struct evhttp_connection *evcon) if (evhttp_connected(evcon) && evcon->closecb != NULL) (*evcon->closecb)(evcon, evcon->closecb_arg); - /* if we have a bufferevent factory callback set, get a new bufferevent */ - if (NULL != evcon->bufcb && -1 != bufferevent_getfd(evcon->bufev)) { - struct bufferevent *bev = (*evcon->bufcb)(evcon->bufcb_arg); - - if (NULL == bev) { - event_warn("%s: bufferevent factory callback failed", __func__); - } - else { - if (bufferevent_get_base(bev) != evcon->base) { - bufferevent_base_set(evcon->base, bev); - } - - bufferevent_free(evcon->bufev); - evcon->bufev = bev; - } - } - + /* if we have a bufferevent factory callback set, get a new bufferevent */ + if (NULL != evcon->bufcb && -1 != bufferevent_getfd(evcon->bufev)) { + struct bufferevent *bev = (*evcon->bufcb)(evcon->bufcb_arg); + + if (NULL == bev) { + event_warn("%s: bufferevent factory callback failed", __func__); + } + else { + if (bufferevent_get_base(bev) != evcon->base) { + bufferevent_base_set(evcon->base, bev); + } + + bufferevent_free(evcon->bufev); + evcon->bufev = bev; + } + } + shutdown(evcon->fd, EVUTIL_SHUT_WR); evutil_closesocket(evcon->fd); evcon->fd = -1; @@ -1585,7 +1585,7 @@ evhttp_error_cb(struct bufferevent *bufev, short what, void *arg) } evhttp_connection_fail_(evcon, EVREQ_HTTP_EOF); - } else if (what == BEV_EVENT_CONNECTED) { + } else if (what == BEV_EVENT_CONNECTED) { } else { evhttp_connection_fail_(evcon, EVREQ_HTTP_BUFFER_ERROR); } @@ -1918,13 +1918,13 @@ evhttp_parse_request_line(struct evhttp_request *req, char *line, size_t len) !evhttp_find_vhost(req->evcon->http_server, NULL, hostname)) req->flags |= EVHTTP_PROXY_REQUEST; - { - struct evhttp_cb * chunkcb = evhttp_dispatch_callback(&req->evcon->http_server->callbacks, req, 1); - if (chunkcb) { - req->chunk_cb = chunkcb->cb; - req->chunk_cb_arg = chunkcb->cbarg; - } - } + { + struct evhttp_cb * chunkcb = evhttp_dispatch_callback(&req->evcon->http_server->callbacks, req, 1); + if (chunkcb) { + req->chunk_cb = chunkcb->cb; + req->chunk_cb_arg = chunkcb->cbarg; + } + } return 0; } @@ -2403,30 +2403,30 @@ evhttp_read_header(struct evhttp_connection *evcon, * happen elsewhere. */ -struct evhttp_connection *evhttp_connection_base_bufferevent_factory_new( - struct event_base *base, struct evdns_base *dnsbase, - bev_factory_cb cb, void * arg, const char *address, unsigned short port) -{ - struct bufferevent *bev = NULL; - - if (NULL != cb) { - if (NULL == (bev = (*cb)(arg))) { - event_warn("%s: bufferevent factory callback failed", __func__); - return (NULL); - } - } - - struct evhttp_connection *ret = - evhttp_connection_base_bufferevent_new(base, dnsbase, bev, address, port); - - if (NULL != ret) { - ret->bufcb = cb; - ret->bufcb_arg = arg; - } - - return (ret); -} - +struct evhttp_connection *evhttp_connection_base_bufferevent_factory_new( + struct event_base *base, struct evdns_base *dnsbase, + bev_factory_cb cb, void * arg, const char *address, unsigned short port) +{ + struct bufferevent *bev = NULL; + + if (NULL != cb) { + if (NULL == (bev = (*cb)(arg))) { + event_warn("%s: bufferevent factory callback failed", __func__); + return (NULL); + } + } + + struct evhttp_connection *ret = + evhttp_connection_base_bufferevent_new(base, dnsbase, bev, address, port); + + if (NULL != ret) { + ret->bufcb = cb; + ret->bufcb_arg = arg; + } + + return (ret); +} + struct evhttp_connection * evhttp_connection_new(const char *address, ev_uint16_t port) { @@ -2434,7 +2434,7 @@ evhttp_connection_new(const char *address, ev_uint16_t port) } struct evhttp_connection * -evhttp_connection_base_bufferevent_new(struct event_base *base, struct evdns_base *dnsbase, struct bufferevent* bev, +evhttp_connection_base_bufferevent_new(struct event_base *base, struct evdns_base *dnsbase, struct bufferevent* bev, const char *address, ev_uint16_t port) { struct evhttp_connection *evcon = NULL; @@ -2460,11 +2460,11 @@ evhttp_connection_base_bufferevent_new(struct event_base *base, struct evdns_bas goto error; } - if (bev == NULL) { + if (bev == NULL) { if (!(bev = bufferevent_socket_new(base, -1, 0))) { event_warn("%s: bufferevent_socket_new failed", __func__); - goto error; - } + goto error; + } } bufferevent_setcb(bev, evhttp_read_cb, evhttp_write_cb, evhttp_error_cb, evcon); @@ -2505,17 +2505,17 @@ struct bufferevent* evhttp_connection_get_bufferevent(struct evhttp_connection * struct evhttp * evhttp_connection_get_server(struct evhttp_connection *evcon) -{ +{ return evcon->http_server; } - -struct evhttp_connection * -evhttp_connection_base_new(struct event_base *base, struct evdns_base *dnsbase, + +struct evhttp_connection * +evhttp_connection_base_new(struct event_base *base, struct evdns_base *dnsbase, const char *address, ev_uint16_t port) -{ - return evhttp_connection_base_bufferevent_new(base, dnsbase, NULL, address, port); -} - +{ + return evhttp_connection_base_bufferevent_new(base, dnsbase, NULL, address, port); +} + void evhttp_connection_set_family(struct evhttp_connection *evcon, int family) { @@ -3390,7 +3390,7 @@ evhttp_parse_query_str(const char *uri, struct evkeyvalq *headers) } static struct evhttp_cb * -evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req, int chunked) +evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req, int chunked) { struct evhttp_cb *cb; size_t offset = 0; @@ -3407,13 +3407,13 @@ evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req, TAILQ_FOREACH(cb, callbacks, next) { if (!strcmp(cb->what, translated)) { - if (chunked < 0) { - mm_free(translated); - return (cb); - } else if (chunked == cb->chunked) { - mm_free(translated); - return (cb); - } + if (chunked < 0) { + mm_free(translated); + return (cb); + } else if (chunked == cb->chunked) { + mm_free(translated); + return (cb); + } } } @@ -3554,7 +3554,7 @@ evhttp_handle_request(struct evhttp_request *req, void *arg) evhttp_find_vhost(http, &http, hostname); } - if ((cb = evhttp_dispatch_callback(&http->callbacks, req, -1)) != NULL) { + if ((cb = evhttp_dispatch_callback(&http->callbacks, req, -1)) != NULL) { (*cb->cb)(req, cb->cbarg); return; } @@ -3974,8 +3974,8 @@ evhttp_set_allowed_methods(struct evhttp* http, ev_uint16_t methods) } int -evhttp_set_cb_internal(struct evhttp *http, const char *uri, - void (*cb)(struct evhttp_request *, void *), void *cbarg, int chunked) +evhttp_set_cb_internal(struct evhttp *http, const char *uri, + void (*cb)(struct evhttp_request *, void *), void *cbarg, int chunked) { struct evhttp_cb *http_cb; @@ -3997,7 +3997,7 @@ evhttp_set_cb_internal(struct evhttp *http, const char *uri, } http_cb->cb = cb; http_cb->cbarg = cbarg; - http_cb->chunked = chunked; + http_cb->chunked = chunked; TAILQ_INSERT_TAIL(&http->callbacks, http_cb, next); @@ -4005,19 +4005,19 @@ evhttp_set_cb_internal(struct evhttp *http, const char *uri, } int -evhttp_set_cb(struct evhttp *http, const char *uri, - void (*cb)(struct evhttp_request *, void *), void *cbarg) -{ - return evhttp_set_cb_internal(http, uri, cb, cbarg, 0); -} - -int evhttp_set_chunk_cb(struct evhttp *http, const char *path, - void (*chunk_cb)(struct evhttp_request *, void *), void *cb_arg) -{ - return evhttp_set_cb_internal(http, path, chunk_cb, cb_arg, 1); -} - -int +evhttp_set_cb(struct evhttp *http, const char *uri, + void (*cb)(struct evhttp_request *, void *), void *cbarg) +{ + return evhttp_set_cb_internal(http, uri, cb, cbarg, 0); +} + +int evhttp_set_chunk_cb(struct evhttp *http, const char *path, + void (*chunk_cb)(struct evhttp_request *, void *), void *cb_arg) +{ + return evhttp_set_cb_internal(http, path, chunk_cb, cb_arg, 1); +} + +int evhttp_del_cb(struct evhttp *http, const char *uri) { struct evhttp_cb *http_cb; @@ -4044,14 +4044,14 @@ evhttp_set_gencb(struct evhttp *http, http->gencbarg = cbarg; } -void -evhttp_set_bevcb(struct evhttp *http, - struct bufferevent* (*cb)(struct event_base *, void *), void *cbarg) -{ - http->bevcb = cb; - http->bevcbarg = cbarg; -} - +void +evhttp_set_bevcb(struct evhttp *http, + struct bufferevent* (*cb)(struct event_base *, void *), void *cbarg) +{ + http->bevcb = cb; + http->bevcbarg = cbarg; +} + /* * Request related functions */ @@ -4166,10 +4166,10 @@ evhttp_connection_get_base(struct evhttp_connection *conn) void evhttp_request_set_chunked_cb(struct evhttp_request *req, - void (*cb)(struct evhttp_request *, void *), void *arg) + void (*cb)(struct evhttp_request *, void *), void *arg) { req->chunk_cb = cb; - req->chunk_cb_arg = arg; + req->chunk_cb_arg = arg; } void @@ -4305,7 +4305,7 @@ evhttp_get_request_connection( { struct evhttp_connection *evcon; char *hostname = NULL, *portname = NULL; - struct bufferevent* bev = NULL; + struct bufferevent* bev = NULL; #ifdef EVENT__HAVE_STRUCT_SOCKADDR_UN if (sa->sa_family == AF_UNIX) { @@ -4325,11 +4325,11 @@ evhttp_get_request_connection( __func__, hostname, portname, EV_SOCK_ARG(fd))); /* we need a connection object to put the http request on */ - if (http->bevcb != NULL) { - bev = (*http->bevcb)(http->base, http->bevcbarg); - } - evcon = evhttp_connection_base_bufferevent_new( - http->base, NULL, bev, hostname, atoi(portname)); + if (http->bevcb != NULL) { + bev = (*http->bevcb)(http->base, http->bevcbarg); + } + evcon = evhttp_connection_base_bufferevent_new( + http->base, NULL, bev, hostname, atoi(portname)); mm_free(hostname); mm_free(portname); if (evcon == NULL) diff --git a/contrib/libs/libevent/include/event2/http.h b/contrib/libs/libevent/include/event2/http.h index c98cd98f62..40d411ddd2 100644 --- a/contrib/libs/libevent/include/event2/http.h +++ b/contrib/libs/libevent/include/event2/http.h @@ -38,7 +38,7 @@ extern "C" { /* In case we haven't included the right headers yet. */ struct evbuffer; struct event_base; -struct bufferevent; +struct bufferevent; struct evhttp_connection; /** @file event2/http.h @@ -72,7 +72,7 @@ struct evhttp_request; struct evkeyvalq; struct evhttp_bound_socket; struct evconnlistener; -struct evdns_base; +struct evdns_base; /** * Create a new HTTP server. @@ -265,9 +265,9 @@ int evhttp_set_cb(struct evhttp *http, const char *path, void (*cb)(struct evhttp_request *, void *), void *cb_arg); EVENT2_EXPORT_SYMBOL -int evhttp_set_chunk_cb(struct evhttp *http, const char *path, - void (*chunk_cb)(struct evhttp_request *, void *), void *cb_arg); - +int evhttp_set_chunk_cb(struct evhttp *http, const char *path, + void (*chunk_cb)(struct evhttp_request *, void *), void *cb_arg); + /** Removes the callback for a specified URI */ EVENT2_EXPORT_SYMBOL int evhttp_del_cb(struct evhttp *, const char *); @@ -302,9 +302,9 @@ void evhttp_set_gencb(struct evhttp *http, @param arg an context argument for the callback */ EVENT2_EXPORT_SYMBOL -void evhttp_set_bevcb(struct evhttp *http, +void evhttp_set_bevcb(struct evhttp *http, struct bufferevent *(*cb)(struct event_base *, void *), void *arg); - + /** Adds a virtual host to the http server. @@ -537,38 +537,38 @@ struct evhttp_connection *evhttp_connection_base_bufferevent_new( struct event_base *base, struct evdns_base *dnsbase, struct bufferevent* bev, const char *address, ev_uint16_t port); /** - * Creates and returns a new bufferevent object. - */ -typedef struct bufferevent* (*bev_factory_cb)(void *); - -/** - * Create and return a connection object that can be used for making HTTP - * requests. The connection object tries to resolve address and establish the - * connection when it is given an http request object. The specified factory - * function is called with the user-supplied argument to retrieve a new - * bufferevent whenever the underlying HTTP connection needs to be - * reestablished. This is what you want if, for example, you have a bufferevent - * that needs to perform some setup for new connections, such as an SSL - * bufferevent. - * - * @param base the event_base to use for handling the connection - * @param dnsbase the dns_base to use for resolving host names; if not - * specified host name resolution will block. - * @param cb a callback that returns a new bufferevent to use for connecting to - * the server; if NULL, behavior is the same as in calling - * evhttp_connection_base_bufferevent_new with a NULL bufferevent. The - * returned bufferevents will be freed as necessary. The returned - * bufferevents must have no fd set on them. - * @param arg the argument to supply to the callback - * @param address the address to which to connect - * @param port the port to connect to - * @return an evhttp_connection object that can be used for making requests - */ -struct evhttp_connection *evhttp_connection_base_bufferevent_factory_new( - struct event_base *base, struct evdns_base *dnsbase, - bev_factory_cb cb, void * arg, const char *address, unsigned short port); - -/** + * Creates and returns a new bufferevent object. + */ +typedef struct bufferevent* (*bev_factory_cb)(void *); + +/** + * Create and return a connection object that can be used for making HTTP + * requests. The connection object tries to resolve address and establish the + * connection when it is given an http request object. The specified factory + * function is called with the user-supplied argument to retrieve a new + * bufferevent whenever the underlying HTTP connection needs to be + * reestablished. This is what you want if, for example, you have a bufferevent + * that needs to perform some setup for new connections, such as an SSL + * bufferevent. + * + * @param base the event_base to use for handling the connection + * @param dnsbase the dns_base to use for resolving host names; if not + * specified host name resolution will block. + * @param cb a callback that returns a new bufferevent to use for connecting to + * the server; if NULL, behavior is the same as in calling + * evhttp_connection_base_bufferevent_new with a NULL bufferevent. The + * returned bufferevents will be freed as necessary. The returned + * bufferevents must have no fd set on them. + * @param arg the argument to supply to the callback + * @param address the address to which to connect + * @param port the port to connect to + * @return an evhttp_connection object that can be used for making requests + */ +struct evhttp_connection *evhttp_connection_base_bufferevent_factory_new( + struct event_base *base, struct evdns_base *dnsbase, + bev_factory_cb cb, void * arg, const char *address, unsigned short port); + +/** * Return the bufferevent that an evhttp_connection is using. */ EVENT2_EXPORT_SYMBOL diff --git a/contrib/libs/libevent/include/event2/http_struct.h b/contrib/libs/libevent/include/event2/http_struct.h index ff2538354b..2691c8c037 100644 --- a/contrib/libs/libevent/include/event2/http_struct.h +++ b/contrib/libs/libevent/include/event2/http_struct.h @@ -78,8 +78,8 @@ struct { /** The request should be freed upstack */ #define EVHTTP_REQ_NEEDS_FREE 0x0010 -#define EVHTTP_REQ_PROCESS_CHUNKS_END 0x0024 - +#define EVHTTP_REQ_PROCESS_CHUNKS_END 0x0024 + struct evkeyvalq *input_headers; struct evkeyvalq *output_headers; @@ -122,7 +122,7 @@ struct { * the regular callback. */ void (*chunk_cb)(struct evhttp_request *, void *); - void *chunk_cb_arg; + void *chunk_cb_arg; /* * Callback added for forked-daapd so they can collect ICY diff --git a/contrib/libs/pcre/pcre_scanner.cc b/contrib/libs/pcre/pcre_scanner.cc index 3715dc161f..30400294d2 100644 --- a/contrib/libs/pcre/pcre_scanner.cc +++ b/contrib/libs/pcre/pcre_scanner.cc @@ -1,199 +1,199 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifdef HAVE_CONFIG_H +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifdef HAVE_CONFIG_H #include "pcre_config.h" -#endif - -#include <vector> -#include <assert.h> - -#include "pcrecpp_internal.h" -#include "pcre_scanner.h" - -using std::vector; - -namespace pcrecpp { - -Scanner::Scanner() - : data_(), - input_(data_), - skip_(NULL), - should_skip_(false), - skip_repeat_(false), - save_comments_(false), - comments_(NULL), - comments_offset_(0) { -} - -Scanner::Scanner(const string& in) - : data_(in), - input_(data_), - skip_(NULL), - should_skip_(false), - skip_repeat_(false), - save_comments_(false), - comments_(NULL), - comments_offset_(0) { -} - -Scanner::~Scanner() { - delete skip_; - delete comments_; -} - -void Scanner::SetSkipExpression(const char* re) { - delete skip_; - if (re != NULL) { - skip_ = new RE(re); - should_skip_ = true; - skip_repeat_ = true; - ConsumeSkip(); - } else { - skip_ = NULL; - should_skip_ = false; - skip_repeat_ = false; - } -} - -void Scanner::Skip(const char* re) { - delete skip_; - if (re != NULL) { - skip_ = new RE(re); - should_skip_ = true; - skip_repeat_ = false; - ConsumeSkip(); - } else { - skip_ = NULL; - should_skip_ = false; - skip_repeat_ = false; - } -} - -void Scanner::DisableSkip() { - assert(skip_ != NULL); - should_skip_ = false; -} - -void Scanner::EnableSkip() { - assert(skip_ != NULL); - should_skip_ = true; - ConsumeSkip(); -} - -int Scanner::LineNumber() const { - // TODO: Make it more efficient by keeping track of the last point - // where we computed line numbers and counting newlines since then. - // We could use std:count, but not all systems have it. :-( - int count = 1; - for (const char* p = data_.data(); p < input_.data(); ++p) - if (*p == '\n') - ++count; - return count; -} - -int Scanner::Offset() const { - return (int)(input_.data() - data_.c_str()); -} - -bool Scanner::LookingAt(const RE& re) const { - int consumed; - return re.DoMatch(input_, RE::ANCHOR_START, &consumed, 0, 0); -} - - -bool Scanner::Consume(const RE& re, - const Arg& arg0, - const Arg& arg1, - const Arg& arg2) { - const bool result = re.Consume(&input_, arg0, arg1, arg2); - if (result && should_skip_) ConsumeSkip(); - return result; -} - -// helper function to consume *skip_ and honour save_comments_ -void Scanner::ConsumeSkip() { - const char* start_data = input_.data(); - while (skip_->Consume(&input_)) { - if (!skip_repeat_) { - // Only one skip allowed. - break; - } - } - if (save_comments_) { - if (comments_ == NULL) { - comments_ = new vector<StringPiece>; - } - // already pointing one past end, so no need to +1 - int length = (int)(input_.data() - start_data); - if (length > 0) { - comments_->push_back(StringPiece(start_data, length)); - } - } -} - - -void Scanner::GetComments(int start, int end, vector<StringPiece> *ranges) { - // short circuit out if we've not yet initialized comments_ - // (e.g., when save_comments is false) - if (!comments_) { - return; - } - // TODO: if we guarantee that comments_ will contain StringPieces - // that are ordered by their start, then we can do a binary search - // for the first StringPiece at or past start and then scan for the - // ones contained in the range, quit early (use equal_range or - // lower_bound) - for (vector<StringPiece>::const_iterator it = comments_->begin(); - it != comments_->end(); ++it) { - if ((it->data() >= data_.c_str() + start && - it->data() + it->size() <= data_.c_str() + end)) { - ranges->push_back(*it); - } - } -} - - -void Scanner::GetNextComments(vector<StringPiece> *ranges) { - // short circuit out if we've not yet initialized comments_ - // (e.g., when save_comments is false) - if (!comments_) { - return; - } - for (vector<StringPiece>::const_iterator it = - comments_->begin() + comments_offset_; - it != comments_->end(); ++it) { - ranges->push_back(*it); - ++comments_offset_; - } -} - -} // namespace pcrecpp +#endif + +#include <vector> +#include <assert.h> + +#include "pcrecpp_internal.h" +#include "pcre_scanner.h" + +using std::vector; + +namespace pcrecpp { + +Scanner::Scanner() + : data_(), + input_(data_), + skip_(NULL), + should_skip_(false), + skip_repeat_(false), + save_comments_(false), + comments_(NULL), + comments_offset_(0) { +} + +Scanner::Scanner(const string& in) + : data_(in), + input_(data_), + skip_(NULL), + should_skip_(false), + skip_repeat_(false), + save_comments_(false), + comments_(NULL), + comments_offset_(0) { +} + +Scanner::~Scanner() { + delete skip_; + delete comments_; +} + +void Scanner::SetSkipExpression(const char* re) { + delete skip_; + if (re != NULL) { + skip_ = new RE(re); + should_skip_ = true; + skip_repeat_ = true; + ConsumeSkip(); + } else { + skip_ = NULL; + should_skip_ = false; + skip_repeat_ = false; + } +} + +void Scanner::Skip(const char* re) { + delete skip_; + if (re != NULL) { + skip_ = new RE(re); + should_skip_ = true; + skip_repeat_ = false; + ConsumeSkip(); + } else { + skip_ = NULL; + should_skip_ = false; + skip_repeat_ = false; + } +} + +void Scanner::DisableSkip() { + assert(skip_ != NULL); + should_skip_ = false; +} + +void Scanner::EnableSkip() { + assert(skip_ != NULL); + should_skip_ = true; + ConsumeSkip(); +} + +int Scanner::LineNumber() const { + // TODO: Make it more efficient by keeping track of the last point + // where we computed line numbers and counting newlines since then. + // We could use std:count, but not all systems have it. :-( + int count = 1; + for (const char* p = data_.data(); p < input_.data(); ++p) + if (*p == '\n') + ++count; + return count; +} + +int Scanner::Offset() const { + return (int)(input_.data() - data_.c_str()); +} + +bool Scanner::LookingAt(const RE& re) const { + int consumed; + return re.DoMatch(input_, RE::ANCHOR_START, &consumed, 0, 0); +} + + +bool Scanner::Consume(const RE& re, + const Arg& arg0, + const Arg& arg1, + const Arg& arg2) { + const bool result = re.Consume(&input_, arg0, arg1, arg2); + if (result && should_skip_) ConsumeSkip(); + return result; +} + +// helper function to consume *skip_ and honour save_comments_ +void Scanner::ConsumeSkip() { + const char* start_data = input_.data(); + while (skip_->Consume(&input_)) { + if (!skip_repeat_) { + // Only one skip allowed. + break; + } + } + if (save_comments_) { + if (comments_ == NULL) { + comments_ = new vector<StringPiece>; + } + // already pointing one past end, so no need to +1 + int length = (int)(input_.data() - start_data); + if (length > 0) { + comments_->push_back(StringPiece(start_data, length)); + } + } +} + + +void Scanner::GetComments(int start, int end, vector<StringPiece> *ranges) { + // short circuit out if we've not yet initialized comments_ + // (e.g., when save_comments is false) + if (!comments_) { + return; + } + // TODO: if we guarantee that comments_ will contain StringPieces + // that are ordered by their start, then we can do a binary search + // for the first StringPiece at or past start and then scan for the + // ones contained in the range, quit early (use equal_range or + // lower_bound) + for (vector<StringPiece>::const_iterator it = comments_->begin(); + it != comments_->end(); ++it) { + if ((it->data() >= data_.c_str() + start && + it->data() + it->size() <= data_.c_str() + end)) { + ranges->push_back(*it); + } + } +} + + +void Scanner::GetNextComments(vector<StringPiece> *ranges) { + // short circuit out if we've not yet initialized comments_ + // (e.g., when save_comments is false) + if (!comments_) { + return; + } + for (vector<StringPiece>::const_iterator it = + comments_->begin() + comments_offset_; + it != comments_->end(); ++it) { + ranges->push_back(*it); + ++comments_offset_; + } +} + +} // namespace pcrecpp diff --git a/contrib/libs/pcre/pcre_stringpiece.cc b/contrib/libs/pcre/pcre_stringpiece.cc index 3fd01db0ab..b51b5482d0 100644 --- a/contrib/libs/pcre/pcre_stringpiece.cc +++ b/contrib/libs/pcre/pcre_stringpiece.cc @@ -1,43 +1,43 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: wilsonh@google.com (Wilson Hsieh) -// - -#ifdef HAVE_CONFIG_H +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wilsonh@google.com (Wilson Hsieh) +// + +#ifdef HAVE_CONFIG_H #include "pcre_config.h" -#endif - -#include <iostream> -#include "pcrecpp_internal.h" -#include "pcre_stringpiece.h" - -std::ostream& operator<<(std::ostream& o, const pcrecpp::StringPiece& piece) { - return (o << piece.as_string()); -} +#endif + +#include <iostream> +#include "pcrecpp_internal.h" +#include "pcre_stringpiece.h" + +std::ostream& operator<<(std::ostream& o, const pcrecpp::StringPiece& piece) { + return (o << piece.as_string()); +} diff --git a/contrib/libs/pcre/pcre_stringpiece.h b/contrib/libs/pcre/pcre_stringpiece.h index a1dd26df71..cd0a7c9bf4 100644 --- a/contrib/libs/pcre/pcre_stringpiece.h +++ b/contrib/libs/pcre/pcre_stringpiece.h @@ -1,47 +1,47 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// -// A string like object that points into another piece of memory. -// Useful for providing an interface that allows clients to easily -// pass in either a "const char*" or a "string". -// -// Arghh! I wish C++ literals were automatically of type "string". - -#ifndef _PCRE_STRINGPIECE_H -#define _PCRE_STRINGPIECE_H - -#include <cstring> -#include <string> -#include <iosfwd> // for ostream forward-declaration - +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// +// A string like object that points into another piece of memory. +// Useful for providing an interface that allows clients to easily +// pass in either a "const char*" or a "string". +// +// Arghh! I wish C++ literals were automatically of type "string". + +#ifndef _PCRE_STRINGPIECE_H +#define _PCRE_STRINGPIECE_H + +#include <cstring> +#include <string> +#include <iosfwd> // for ostream forward-declaration + #if 0 #define HAVE_TYPE_TRAITS #include <type_traits.h> @@ -49,132 +49,132 @@ #define HAVE_TYPE_TRAITS #error #include <bits/type_traits.h> #endif - + #include "pcre.h" namespace pcrecpp { -using std::memcmp; -using std::strlen; -using std::string; - -class PCRECPP_EXP_DEFN StringPiece { - private: - const char* ptr_; - int length_; - - public: - // We provide non-explicit singleton constructors so users can pass - // in a "const char*" or a "string" wherever a "StringPiece" is - // expected. - StringPiece() - : ptr_(NULL), length_(0) { } - StringPiece(const char* str) - : ptr_(str), length_(static_cast<int>(strlen(ptr_))) { } - StringPiece(const unsigned char* str) - : ptr_(reinterpret_cast<const char*>(str)), - length_(static_cast<int>(strlen(ptr_))) { } - StringPiece(const string& str) - : ptr_(str.data()), length_(static_cast<int>(str.size())) { } - StringPiece(const char* offset, int len) - : ptr_(offset), length_(len) { } - - // data() may return a pointer to a buffer with embedded NULs, and the - // returned buffer may or may not be null terminated. Therefore it is - // typically a mistake to pass data() to a routine that expects a NUL - // terminated string. Use "as_string().c_str()" if you really need to do - // this. Or better yet, change your routine so it does not rely on NUL - // termination. - const char* data() const { return ptr_; } - int size() const { return length_; } - bool empty() const { return length_ == 0; } - - void clear() { ptr_ = NULL; length_ = 0; } - void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } - void set(const char* str) { - ptr_ = str; - length_ = static_cast<int>(strlen(str)); - } - void set(const void* buffer, int len) { - ptr_ = reinterpret_cast<const char*>(buffer); - length_ = len; - } - - char operator[](int i) const { return ptr_[i]; } - - void remove_prefix(int n) { - ptr_ += n; - length_ -= n; - } - - void remove_suffix(int n) { - length_ -= n; - } - - bool operator==(const StringPiece& x) const { - return ((length_ == x.length_) && - (memcmp(ptr_, x.ptr_, length_) == 0)); - } - bool operator!=(const StringPiece& x) const { - return !(*this == x); - } - -#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ - bool operator cmp (const StringPiece& x) const { \ - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ - return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ - } - STRINGPIECE_BINARY_PREDICATE(<, <); - STRINGPIECE_BINARY_PREDICATE(<=, <); - STRINGPIECE_BINARY_PREDICATE(>=, >); - STRINGPIECE_BINARY_PREDICATE(>, >); -#undef STRINGPIECE_BINARY_PREDICATE - - int compare(const StringPiece& x) const { - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); - if (r == 0) { - if (length_ < x.length_) r = -1; - else if (length_ > x.length_) r = +1; - } - return r; - } - - string as_string() const { - return string(data(), size()); - } - - void CopyToString(string* target) const { - target->assign(ptr_, length_); - } - - // Does "this" start with "x" - bool starts_with(const StringPiece& x) const { - return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); - } -}; - -} // namespace pcrecpp - -// ------------------------------------------------------------------ -// Functions used to create STL containers that use StringPiece -// Remember that a StringPiece's lifetime had better be less than -// that of the underlying string or char*. If it is not, then you -// cannot safely store a StringPiece into an STL container -// ------------------------------------------------------------------ - -#ifdef HAVE_TYPE_TRAITS -// This makes vector<StringPiece> really fast for some STL implementations -template<> struct __type_traits<pcrecpp::StringPiece> { - typedef __true_type has_trivial_default_constructor; - typedef __true_type has_trivial_copy_constructor; - typedef __true_type has_trivial_assignment_operator; - typedef __true_type has_trivial_destructor; - typedef __true_type is_POD_type; -}; -#endif - -// allow StringPiece to be logged -PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, - const pcrecpp::StringPiece& piece); - -#endif /* _PCRE_STRINGPIECE_H */ +using std::memcmp; +using std::strlen; +using std::string; + +class PCRECPP_EXP_DEFN StringPiece { + private: + const char* ptr_; + int length_; + + public: + // We provide non-explicit singleton constructors so users can pass + // in a "const char*" or a "string" wherever a "StringPiece" is + // expected. + StringPiece() + : ptr_(NULL), length_(0) { } + StringPiece(const char* str) + : ptr_(str), length_(static_cast<int>(strlen(ptr_))) { } + StringPiece(const unsigned char* str) + : ptr_(reinterpret_cast<const char*>(str)), + length_(static_cast<int>(strlen(ptr_))) { } + StringPiece(const string& str) + : ptr_(str.data()), length_(static_cast<int>(str.size())) { } + StringPiece(const char* offset, int len) + : ptr_(offset), length_(len) { } + + // data() may return a pointer to a buffer with embedded NULs, and the + // returned buffer may or may not be null terminated. Therefore it is + // typically a mistake to pass data() to a routine that expects a NUL + // terminated string. Use "as_string().c_str()" if you really need to do + // this. Or better yet, change your routine so it does not rely on NUL + // termination. + const char* data() const { return ptr_; } + int size() const { return length_; } + bool empty() const { return length_ == 0; } + + void clear() { ptr_ = NULL; length_ = 0; } + void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } + void set(const char* str) { + ptr_ = str; + length_ = static_cast<int>(strlen(str)); + } + void set(const void* buffer, int len) { + ptr_ = reinterpret_cast<const char*>(buffer); + length_ = len; + } + + char operator[](int i) const { return ptr_[i]; } + + void remove_prefix(int n) { + ptr_ += n; + length_ -= n; + } + + void remove_suffix(int n) { + length_ -= n; + } + + bool operator==(const StringPiece& x) const { + return ((length_ == x.length_) && + (memcmp(ptr_, x.ptr_, length_) == 0)); + } + bool operator!=(const StringPiece& x) const { + return !(*this == x); + } + +#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ + bool operator cmp (const StringPiece& x) const { \ + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ + return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ + } + STRINGPIECE_BINARY_PREDICATE(<, <); + STRINGPIECE_BINARY_PREDICATE(<=, <); + STRINGPIECE_BINARY_PREDICATE(>=, >); + STRINGPIECE_BINARY_PREDICATE(>, >); +#undef STRINGPIECE_BINARY_PREDICATE + + int compare(const StringPiece& x) const { + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); + if (r == 0) { + if (length_ < x.length_) r = -1; + else if (length_ > x.length_) r = +1; + } + return r; + } + + string as_string() const { + return string(data(), size()); + } + + void CopyToString(string* target) const { + target->assign(ptr_, length_); + } + + // Does "this" start with "x" + bool starts_with(const StringPiece& x) const { + return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); + } +}; + +} // namespace pcrecpp + +// ------------------------------------------------------------------ +// Functions used to create STL containers that use StringPiece +// Remember that a StringPiece's lifetime had better be less than +// that of the underlying string or char*. If it is not, then you +// cannot safely store a StringPiece into an STL container +// ------------------------------------------------------------------ + +#ifdef HAVE_TYPE_TRAITS +// This makes vector<StringPiece> really fast for some STL implementations +template<> struct __type_traits<pcrecpp::StringPiece> { + typedef __true_type has_trivial_default_constructor; + typedef __true_type has_trivial_copy_constructor; + typedef __true_type has_trivial_assignment_operator; + typedef __true_type has_trivial_destructor; + typedef __true_type is_POD_type; +}; +#endif + +// allow StringPiece to be logged +PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, + const pcrecpp::StringPiece& piece); + +#endif /* _PCRE_STRINGPIECE_H */ diff --git a/contrib/libs/pcre/pcrecpp.cc b/contrib/libs/pcre/pcrecpp.cc index 33d186c7d0..57daa1f59f 100644 --- a/contrib/libs/pcre/pcrecpp.cc +++ b/contrib/libs/pcre/pcrecpp.cc @@ -1,86 +1,86 @@ -// Copyright (c) 2010, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifdef HAVE_CONFIG_H +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifdef HAVE_CONFIG_H #include "pcre_config.h" -#endif - -#include <stdlib.h> -#include <stdio.h> -#include <ctype.h> -#include <limits.h> /* for SHRT_MIN, USHRT_MAX, etc */ -#include <string.h> /* for memcpy */ -#include <assert.h> -#include <errno.h> -#include <string> -#include <algorithm> - -#include "pcrecpp_internal.h" -#include "pcre.h" -#include "pcrecpp.h" -#include "pcre_stringpiece.h" - - -namespace pcrecpp { - -// Maximum number of args we can set -static const int kMaxArgs = 16; -static const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace - -// Special object that stands-in for no argument -Arg RE::no_arg((void*)NULL); - -// This is for ABI compatibility with old versions of pcre (pre-7.6), -// which defined a global no_arg variable instead of putting it in the -// RE class. This works on GCC >= 3, at least. It definitely works -// for ELF, but may not for other object formats (Mach-O, for -// instance, does not support aliases.) We could probably have a more -// inclusive test if we ever needed it. (Note that not only the -// __attribute__ syntax, but also __USER_LABEL_PREFIX__, are -// gnu-specific.) +#endif + +#include <stdlib.h> +#include <stdio.h> +#include <ctype.h> +#include <limits.h> /* for SHRT_MIN, USHRT_MAX, etc */ +#include <string.h> /* for memcpy */ +#include <assert.h> +#include <errno.h> +#include <string> +#include <algorithm> + +#include "pcrecpp_internal.h" +#include "pcre.h" +#include "pcrecpp.h" +#include "pcre_stringpiece.h" + + +namespace pcrecpp { + +// Maximum number of args we can set +static const int kMaxArgs = 16; +static const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace + +// Special object that stands-in for no argument +Arg RE::no_arg((void*)NULL); + +// This is for ABI compatibility with old versions of pcre (pre-7.6), +// which defined a global no_arg variable instead of putting it in the +// RE class. This works on GCC >= 3, at least. It definitely works +// for ELF, but may not for other object formats (Mach-O, for +// instance, does not support aliases.) We could probably have a more +// inclusive test if we ever needed it. (Note that not only the +// __attribute__ syntax, but also __USER_LABEL_PREFIX__, are +// gnu-specific.) #if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) \ && !defined(__INTEL_COMPILER) && !defined(__LCC__) -# define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) -# define ULP_AS_STRING_INTERNAL(x) #x -# define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) -extern Arg no_arg - __attribute__((alias(USER_LABEL_PREFIX_STR "_ZN7pcrecpp2RE6no_argE"))); -#endif - -// If a regular expression has no error, its error_ field points here -static const string empty_string; - -// If the user doesn't ask for any options, we just use this one -static RE_Options default_options; - +# define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) +# define ULP_AS_STRING_INTERNAL(x) #x +# define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) +extern Arg no_arg + __attribute__((alias(USER_LABEL_PREFIX_STR "_ZN7pcrecpp2RE6no_argE"))); +#endif + +// If a regular expression has no error, its error_ field points here +static const string empty_string; + +// If the user doesn't ask for any options, we just use this one +static RE_Options default_options; + // Specials for the start of patterns. See comments where start_options is used // below. (PH June 2018) static const char *start_options[] = { @@ -100,61 +100,61 @@ static const char *start_options[] = { "(*ANY)", "" }; -void RE::Init(const string& pat, const RE_Options* options) { - pattern_ = pat; - if (options == NULL) { - options_ = default_options; - } else { - options_ = *options; - } - error_ = &empty_string; - re_full_ = NULL; - re_partial_ = NULL; - - re_partial_ = Compile(UNANCHORED); - if (re_partial_ != NULL) { - re_full_ = Compile(ANCHOR_BOTH); - } -} - -void RE::Cleanup() { - if (re_full_ != NULL) (*pcre_free)(re_full_); - if (re_partial_ != NULL) (*pcre_free)(re_partial_); - if (error_ != &empty_string) delete error_; -} - - -RE::~RE() { - Cleanup(); -} - - -pcre* RE::Compile(Anchor anchor) { - // First, convert RE_Options into pcre options - int pcre_options = 0; - pcre_options = options_.all_options(); - - // Special treatment for anchoring. This is needed because at - // runtime pcre only provides an option for anchoring at the - // beginning of a string (unless you use offset). - // - // There are three types of anchoring we want: - // UNANCHORED Compile the original pattern, and use - // a pcre unanchored match. - // ANCHOR_START Compile the original pattern, and use - // a pcre anchored match. - // ANCHOR_BOTH Tack a "\z" to the end of the original pattern - // and use a pcre anchored match. - - const char* compile_error; - int eoffset; - pcre* re; - if (anchor != ANCHOR_BOTH) { - re = pcre_compile(pattern_.c_str(), pcre_options, - &compile_error, &eoffset, NULL); - } else { - // Tack a '\z' at the end of RE. Parenthesize it first so that - // the '\z' applies to all top-level alternatives in the regexp. +void RE::Init(const string& pat, const RE_Options* options) { + pattern_ = pat; + if (options == NULL) { + options_ = default_options; + } else { + options_ = *options; + } + error_ = &empty_string; + re_full_ = NULL; + re_partial_ = NULL; + + re_partial_ = Compile(UNANCHORED); + if (re_partial_ != NULL) { + re_full_ = Compile(ANCHOR_BOTH); + } +} + +void RE::Cleanup() { + if (re_full_ != NULL) (*pcre_free)(re_full_); + if (re_partial_ != NULL) (*pcre_free)(re_partial_); + if (error_ != &empty_string) delete error_; +} + + +RE::~RE() { + Cleanup(); +} + + +pcre* RE::Compile(Anchor anchor) { + // First, convert RE_Options into pcre options + int pcre_options = 0; + pcre_options = options_.all_options(); + + // Special treatment for anchoring. This is needed because at + // runtime pcre only provides an option for anchoring at the + // beginning of a string (unless you use offset). + // + // There are three types of anchoring we want: + // UNANCHORED Compile the original pattern, and use + // a pcre unanchored match. + // ANCHOR_START Compile the original pattern, and use + // a pcre anchored match. + // ANCHOR_BOTH Tack a "\z" to the end of the original pattern + // and use a pcre anchored match. + + const char* compile_error; + int eoffset; + pcre* re; + if (anchor != ANCHOR_BOTH) { + re = pcre_compile(pattern_.c_str(), pcre_options, + &compile_error, &eoffset, NULL); + } else { + // Tack a '\z' at the end of RE. Parenthesize it first so that + // the '\z' applies to all top-level alternatives in the regexp. /* When this code was written (for PCRE 6.0) it was enough just to parenthesize the entire pattern. Unfortunately, when the feature of @@ -198,789 +198,789 @@ pcre* RE::Compile(Anchor anchor) { // Wrap the rest of the pattern. wrapped += "(?:"; // A non-counting grouping operator - wrapped += pattern_; - wrapped += ")\\z"; - re = pcre_compile(wrapped.c_str(), pcre_options, - &compile_error, &eoffset, NULL); - } - if (re == NULL) { - if (error_ == &empty_string) error_ = new string(compile_error); - } - return re; -} - -/***** Matching interfaces *****/ - -bool RE::FullMatch(const StringPiece& text, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - return DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize); -} - -bool RE::PartialMatch(const StringPiece& text, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - return DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize); -} - -bool RE::Consume(StringPiece* input, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - if (DoMatchImpl(*input, ANCHOR_START, &consumed, - args, n, vec, kVecSize)) { - input->remove_prefix(consumed); - return true; - } else { - return false; - } -} - -bool RE::FindAndConsume(StringPiece* input, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - if (DoMatchImpl(*input, UNANCHORED, &consumed, - args, n, vec, kVecSize)) { - input->remove_prefix(consumed); - return true; - } else { - return false; - } -} - -bool RE::Replace(const StringPiece& rewrite, - string *str) const { - int vec[kVecSize]; - int matches = TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize); - if (matches == 0) - return false; - - string s; - if (!Rewrite(&s, rewrite, *str, vec, matches)) - return false; - - assert(vec[0] >= 0); - assert(vec[1] >= 0); - str->replace(vec[0], vec[1] - vec[0], s); - return true; -} - -// Returns PCRE_NEWLINE_CRLF, PCRE_NEWLINE_CR, or PCRE_NEWLINE_LF. -// Note that PCRE_NEWLINE_CRLF is defined to be P_N_CR | P_N_LF. -// Modified by PH to add PCRE_NEWLINE_ANY and PCRE_NEWLINE_ANYCRLF. - -static int NewlineMode(int pcre_options) { - // TODO: if we can make it threadsafe, cache this var - int newline_mode = 0; - /* if (newline_mode) return newline_mode; */ // do this once it's cached - if (pcre_options & (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| - PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)) { - newline_mode = (pcre_options & - (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| - PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)); - } else { - int newline; - pcre_config(PCRE_CONFIG_NEWLINE, &newline); - if (newline == 10) - newline_mode = PCRE_NEWLINE_LF; - else if (newline == 13) - newline_mode = PCRE_NEWLINE_CR; - else if (newline == 3338) - newline_mode = PCRE_NEWLINE_CRLF; - else if (newline == -1) - newline_mode = PCRE_NEWLINE_ANY; - else if (newline == -2) - newline_mode = PCRE_NEWLINE_ANYCRLF; - else - assert(NULL == "Unexpected return value from pcre_config(NEWLINE)"); - } - return newline_mode; -} - -int RE::GlobalReplace(const StringPiece& rewrite, - string *str) const { - int count = 0; - int vec[kVecSize]; - string out; - int start = 0; - bool last_match_was_empty_string = false; - - while (start <= static_cast<int>(str->length())) { - // If the previous match was for the empty string, we shouldn't - // just match again: we'll match in the same way and get an - // infinite loop. Instead, we do the match in a special way: - // anchored -- to force another try at the same position -- - // and with a flag saying that this time, ignore empty matches. - // If this special match returns, that means there's a non-empty - // match at this position as well, and we can continue. If not, - // we do what perl does, and just advance by one. - // Notice that perl prints '@@@' for this; - // perl -le '$_ = "aa"; s/b*|aa/@/g; print' - int matches; - if (last_match_was_empty_string) { - matches = TryMatch(*str, start, ANCHOR_START, false, vec, kVecSize); - if (matches <= 0) { - int matchend = start + 1; // advance one character. - // If the current char is CR and we're in CRLF mode, skip LF too. - // Note it's better to call pcre_fullinfo() than to examine - // all_options(), since options_ could have changed bewteen - // compile-time and now, but this is simpler and safe enough. - // Modified by PH to add ANY and ANYCRLF. - if (matchend < static_cast<int>(str->length()) && - (*str)[start] == '\r' && (*str)[matchend] == '\n' && - (NewlineMode(options_.all_options()) == PCRE_NEWLINE_CRLF || - NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANY || - NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANYCRLF)) { - matchend++; - } - // We also need to advance more than one char if we're in utf8 mode. + wrapped += pattern_; + wrapped += ")\\z"; + re = pcre_compile(wrapped.c_str(), pcre_options, + &compile_error, &eoffset, NULL); + } + if (re == NULL) { + if (error_ == &empty_string) error_ = new string(compile_error); + } + return re; +} + +/***** Matching interfaces *****/ + +bool RE::FullMatch(const StringPiece& text, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + return DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize); +} + +bool RE::PartialMatch(const StringPiece& text, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + return DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize); +} + +bool RE::Consume(StringPiece* input, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + if (DoMatchImpl(*input, ANCHOR_START, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE::FindAndConsume(StringPiece* input, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + if (DoMatchImpl(*input, UNANCHORED, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE::Replace(const StringPiece& rewrite, + string *str) const { + int vec[kVecSize]; + int matches = TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + + string s; + if (!Rewrite(&s, rewrite, *str, vec, matches)) + return false; + + assert(vec[0] >= 0); + assert(vec[1] >= 0); + str->replace(vec[0], vec[1] - vec[0], s); + return true; +} + +// Returns PCRE_NEWLINE_CRLF, PCRE_NEWLINE_CR, or PCRE_NEWLINE_LF. +// Note that PCRE_NEWLINE_CRLF is defined to be P_N_CR | P_N_LF. +// Modified by PH to add PCRE_NEWLINE_ANY and PCRE_NEWLINE_ANYCRLF. + +static int NewlineMode(int pcre_options) { + // TODO: if we can make it threadsafe, cache this var + int newline_mode = 0; + /* if (newline_mode) return newline_mode; */ // do this once it's cached + if (pcre_options & (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| + PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)) { + newline_mode = (pcre_options & + (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| + PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)); + } else { + int newline; + pcre_config(PCRE_CONFIG_NEWLINE, &newline); + if (newline == 10) + newline_mode = PCRE_NEWLINE_LF; + else if (newline == 13) + newline_mode = PCRE_NEWLINE_CR; + else if (newline == 3338) + newline_mode = PCRE_NEWLINE_CRLF; + else if (newline == -1) + newline_mode = PCRE_NEWLINE_ANY; + else if (newline == -2) + newline_mode = PCRE_NEWLINE_ANYCRLF; + else + assert(NULL == "Unexpected return value from pcre_config(NEWLINE)"); + } + return newline_mode; +} + +int RE::GlobalReplace(const StringPiece& rewrite, + string *str) const { + int count = 0; + int vec[kVecSize]; + string out; + int start = 0; + bool last_match_was_empty_string = false; + + while (start <= static_cast<int>(str->length())) { + // If the previous match was for the empty string, we shouldn't + // just match again: we'll match in the same way and get an + // infinite loop. Instead, we do the match in a special way: + // anchored -- to force another try at the same position -- + // and with a flag saying that this time, ignore empty matches. + // If this special match returns, that means there's a non-empty + // match at this position as well, and we can continue. If not, + // we do what perl does, and just advance by one. + // Notice that perl prints '@@@' for this; + // perl -le '$_ = "aa"; s/b*|aa/@/g; print' + int matches; + if (last_match_was_empty_string) { + matches = TryMatch(*str, start, ANCHOR_START, false, vec, kVecSize); + if (matches <= 0) { + int matchend = start + 1; // advance one character. + // If the current char is CR and we're in CRLF mode, skip LF too. + // Note it's better to call pcre_fullinfo() than to examine + // all_options(), since options_ could have changed bewteen + // compile-time and now, but this is simpler and safe enough. + // Modified by PH to add ANY and ANYCRLF. + if (matchend < static_cast<int>(str->length()) && + (*str)[start] == '\r' && (*str)[matchend] == '\n' && + (NewlineMode(options_.all_options()) == PCRE_NEWLINE_CRLF || + NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANY || + NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANYCRLF)) { + matchend++; + } + // We also need to advance more than one char if we're in utf8 mode. #ifdef SUPPORT_UTF - if (options_.utf8()) { - while (matchend < static_cast<int>(str->length()) && - ((*str)[matchend] & 0xc0) == 0x80) - matchend++; - } -#endif - if (start < static_cast<int>(str->length())) - out.append(*str, start, matchend - start); - start = matchend; - last_match_was_empty_string = false; - continue; - } - } else { - matches = TryMatch(*str, start, UNANCHORED, true, vec, kVecSize); - if (matches <= 0) - break; - } - int matchstart = vec[0], matchend = vec[1]; - assert(matchstart >= start); - assert(matchend >= matchstart); - out.append(*str, start, matchstart - start); - Rewrite(&out, rewrite, *str, vec, matches); - start = matchend; - count++; - last_match_was_empty_string = (matchstart == matchend); - } - - if (count == 0) - return 0; - - if (start < static_cast<int>(str->length())) - out.append(*str, start, str->length() - start); - swap(out, *str); - return count; -} - -bool RE::Extract(const StringPiece& rewrite, - const StringPiece& text, - string *out) const { - int vec[kVecSize]; - int matches = TryMatch(text, 0, UNANCHORED, true, vec, kVecSize); - if (matches == 0) - return false; - out->erase(); - return Rewrite(out, rewrite, text, vec, matches); -} - -/*static*/ string RE::QuoteMeta(const StringPiece& unquoted) { - string result; - - // Escape any ascii character not in [A-Za-z_0-9]. - // - // Note that it's legal to escape a character even if it has no - // special meaning in a regular expression -- so this function does - // that. (This also makes it identical to the perl function of the - // same name; see `perldoc -f quotemeta`.) The one exception is - // escaping NUL: rather than doing backslash + NUL, like perl does, - // we do '\0', because pcre itself doesn't take embedded NUL chars. - for (int ii = 0; ii < unquoted.size(); ++ii) { - // Note that using 'isalnum' here raises the benchmark time from - // 32ns to 58ns: - if (unquoted[ii] == '\0') { - result += "\\0"; - } else if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && - (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && - (unquoted[ii] < '0' || unquoted[ii] > '9') && - unquoted[ii] != '_' && - // If this is the part of a UTF8 or Latin1 character, we need - // to copy this byte without escaping. Experimentally this is - // what works correctly with the regexp library. - !(unquoted[ii] & 128)) { - result += '\\'; - result += unquoted[ii]; - } else { - result += unquoted[ii]; - } - } - - return result; -} - -/***** Actual matching and rewriting code *****/ - -int RE::TryMatch(const StringPiece& text, - int startpos, - Anchor anchor, - bool empty_ok, - int *vec, - int vecsize) const { - pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_; - if (re == NULL) { - //fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str()); - return 0; - } - - pcre_extra extra = { 0, 0, 0, 0, 0, 0, 0, 0 }; - if (options_.match_limit() > 0) { - extra.flags |= PCRE_EXTRA_MATCH_LIMIT; - extra.match_limit = options_.match_limit(); - } - if (options_.match_limit_recursion() > 0) { - extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - extra.match_limit_recursion = options_.match_limit_recursion(); - } - - // int options = 0; - // Changed by PH as a result of bugzilla #1288 - int options = (options_.all_options() & PCRE_NO_UTF8_CHECK); - - if (anchor != UNANCHORED) - options |= PCRE_ANCHORED; - if (!empty_ok) - options |= PCRE_NOTEMPTY; - - int rc = pcre_exec(re, // The regular expression object - &extra, - (text.data() == NULL) ? "" : text.data(), - text.size(), - startpos, - options, - vec, - vecsize); - - // Handle errors - if (rc == PCRE_ERROR_NOMATCH) { - return 0; - } else if (rc < 0) { - //fprintf(stderr, "Unexpected return code: %d when matching '%s'\n", - // re, pattern_.c_str()); - return 0; - } else if (rc == 0) { - // pcre_exec() returns 0 as a special case when the number of - // capturing subpatterns exceeds the size of the vector. - // When this happens, there is a match and the output vector - // is filled, but we miss out on the positions of the extra subpatterns. - rc = vecsize / 2; - } - - return rc; -} - -bool RE::DoMatchImpl(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const* args, - int n, - int* vec, - int vecsize) const { - assert((1 + n) * 3 <= vecsize); // results + PCRE workspace - int matches = TryMatch(text, 0, anchor, true, vec, vecsize); - assert(matches >= 0); // TryMatch never returns negatives - if (matches == 0) - return false; - - *consumed = vec[1]; - - if (n == 0 || args == NULL) { - // We are not interested in results - return true; - } - - if (NumberOfCapturingGroups() < n) { - // RE has fewer capturing groups than number of arg pointers passed in - return false; - } - - // If we got here, we must have matched the whole pattern. - // We do not need (can not do) any more checks on the value of 'matches' here - // -- see the comment for TryMatch. - for (int i = 0; i < n; i++) { - const int start = vec[2*(i+1)]; - const int limit = vec[2*(i+1)+1]; - if (!args[i]->Parse(text.data() + start, limit-start)) { - // TODO: Should we indicate what the error was? - return false; - } - } - - return true; -} - -bool RE::DoMatch(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const args[], - int n) const { - assert(n >= 0); - size_t const vecsize = (1 + n) * 3; // results + PCRE workspace - // (as for kVecSize) - int space[21]; // use stack allocation for small vecsize (common case) - int* vec = vecsize <= 21 ? space : new int[vecsize]; - bool retval = DoMatchImpl(text, anchor, consumed, args, n, vec, (int)vecsize); - if (vec != space) delete [] vec; - return retval; -} - -bool RE::Rewrite(string *out, const StringPiece &rewrite, - const StringPiece &text, int *vec, int veclen) const { - for (const char *s = rewrite.data(), *end = s + rewrite.size(); - s < end; s++) { - int c = *s; - if (c == '\\') { - c = *++s; - if (isdigit(c)) { - int n = (c - '0'); - if (n >= veclen) { - //fprintf(stderr, requested group %d in regexp %.*s\n", - // n, rewrite.size(), rewrite.data()); - return false; - } - int start = vec[2 * n]; - if (start >= 0) - out->append(text.data() + start, vec[2 * n + 1] - start); - } else if (c == '\\') { - *out += '\\'; - } else { - //fprintf(stderr, "invalid rewrite pattern: %.*s\n", - // rewrite.size(), rewrite.data()); - return false; - } - } else { - *out += c; - } - } - return true; -} - -// Return the number of capturing subpatterns, or -1 if the -// regexp wasn't valid on construction. -int RE::NumberOfCapturingGroups() const { - if (re_partial_ == NULL) return -1; - - int result; - int pcre_retval = pcre_fullinfo(re_partial_, // The regular expression object - NULL, // We did not study the pattern - PCRE_INFO_CAPTURECOUNT, - &result); - assert(pcre_retval == 0); - return result; -} - -/***** Parsers for various types *****/ - -bool Arg::parse_null(const char* str, int n, void* dest) { - (void)str; - (void)n; - // We fail if somebody asked us to store into a non-NULL void* pointer - return (dest == NULL); -} - -bool Arg::parse_string(const char* str, int n, void* dest) { - if (dest == NULL) return true; - reinterpret_cast<string*>(dest)->assign(str, n); - return true; -} - -bool Arg::parse_stringpiece(const char* str, int n, void* dest) { - if (dest == NULL) return true; - reinterpret_cast<StringPiece*>(dest)->set(str, n); - return true; -} - -bool Arg::parse_char(const char* str, int n, void* dest) { - if (n != 1) return false; - if (dest == NULL) return true; - *(reinterpret_cast<char*>(dest)) = str[0]; - return true; -} - -bool Arg::parse_uchar(const char* str, int n, void* dest) { - if (n != 1) return false; - if (dest == NULL) return true; - *(reinterpret_cast<unsigned char*>(dest)) = str[0]; - return true; -} - -// Largest number spec that we are willing to parse -static const int kMaxNumberLength = 32; - -// REQUIRES "buf" must have length at least kMaxNumberLength+1 -// REQUIRES "n > 0" -// Copies "str" into "buf" and null-terminates if necessary. -// Returns one of: -// a. "str" if no termination is needed -// b. "buf" if the string was copied and null-terminated -// c. "" if the input was invalid and has no hope of being parsed -static const char* TerminateNumber(char* buf, const char* str, int n) { - if ((n > 0) && isspace(*str)) { - // We are less forgiving than the strtoxxx() routines and do not - // allow leading spaces. - return ""; - } - - // See if the character right after the input text may potentially - // look like a digit. - if (isdigit(str[n]) || - ((str[n] >= 'a') && (str[n] <= 'f')) || - ((str[n] >= 'A') && (str[n] <= 'F'))) { - if (n > kMaxNumberLength) return ""; // Input too big to be a valid number - memcpy(buf, str, n); - buf[n] = '\0'; - return buf; - } else { - // We can parse right out of the supplied string, so return it. - return str; - } -} - -bool Arg::parse_long_radix(const char* str, - int n, - void* dest, - int radix) { - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - char* end; - errno = 0; - long r = strtol(str, &end, radix); - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast<long*>(dest)) = r; - return true; -} - -bool Arg::parse_ulong_radix(const char* str, - int n, - void* dest, - int radix) { - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - if (str[0] == '-') return false; // strtoul() on a negative number?! - char* end; - errno = 0; - unsigned long r = strtoul(str, &end, radix); - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast<unsigned long*>(dest)) = r; - return true; -} - -bool Arg::parse_short_radix(const char* str, - int n, - void* dest, - int radix) { - long r; - if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse - if (r < SHRT_MIN || r > SHRT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast<short*>(dest)) = static_cast<short>(r); - return true; -} - -bool Arg::parse_ushort_radix(const char* str, - int n, - void* dest, - int radix) { - unsigned long r; - if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse - if (r > USHRT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast<unsigned short*>(dest)) = static_cast<unsigned short>(r); - return true; -} - -bool Arg::parse_int_radix(const char* str, - int n, - void* dest, - int radix) { - long r; - if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse - if (r < INT_MIN || r > INT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast<int*>(dest)) = r; - return true; -} - -bool Arg::parse_uint_radix(const char* str, - int n, - void* dest, - int radix) { - unsigned long r; - if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse - if (r > UINT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast<unsigned int*>(dest)) = r; - return true; -} - -bool Arg::parse_longlong_radix(const char* str, - int n, - void* dest, - int radix) { -#ifndef HAVE_LONG_LONG - return false; -#else - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - char* end; - errno = 0; -#if defined HAVE_STRTOQ - long long r = strtoq(str, &end, radix); -#elif defined HAVE_STRTOLL - long long r = strtoll(str, &end, radix); -#elif defined HAVE__STRTOI64 - long long r = _strtoi64(str, &end, radix); -#elif defined HAVE_STRTOIMAX - long long r = strtoimax(str, &end, radix); -#else -#error parse_longlong_radix: cannot convert input to a long-long -#endif - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast<long long*>(dest)) = r; - return true; -#endif /* HAVE_LONG_LONG */ -} - -bool Arg::parse_ulonglong_radix(const char* str, - int n, - void* dest, - int radix) { -#ifndef HAVE_UNSIGNED_LONG_LONG - return false; -#else - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - if (str[0] == '-') return false; // strtoull() on a negative number?! - char* end; - errno = 0; -#if defined HAVE_STRTOQ - unsigned long long r = strtouq(str, &end, radix); -#elif defined HAVE_STRTOLL - unsigned long long r = strtoull(str, &end, radix); -#elif defined HAVE__STRTOI64 - unsigned long long r = _strtoui64(str, &end, radix); -#elif defined HAVE_STRTOIMAX - unsigned long long r = strtoumax(str, &end, radix); -#else -#error parse_ulonglong_radix: cannot convert input to a long-long -#endif - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast<unsigned long long*>(dest)) = r; - return true; -#endif /* HAVE_UNSIGNED_LONG_LONG */ -} - -bool Arg::parse_double(const char* str, int n, void* dest) { - if (n == 0) return false; - static const int kMaxLength = 200; - char buf[kMaxLength]; - if (n >= kMaxLength) return false; - memcpy(buf, str, n); - buf[n] = '\0'; - errno = 0; - char* end; - double r = strtod(buf, &end); - if (end != buf + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast<double*>(dest)) = r; - return true; -} - -bool Arg::parse_float(const char* str, int n, void* dest) { - double r; - if (!parse_double(str, n, &r)) return false; - if (dest == NULL) return true; - *(reinterpret_cast<float*>(dest)) = static_cast<float>(r); - return true; -} - - -#define DEFINE_INTEGER_PARSERS(name) \ - bool Arg::parse_##name(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 10); \ - } \ - bool Arg::parse_##name##_hex(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 16); \ - } \ - bool Arg::parse_##name##_octal(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 8); \ - } \ - bool Arg::parse_##name##_cradix(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 0); \ - } - -DEFINE_INTEGER_PARSERS(short) /* */ -DEFINE_INTEGER_PARSERS(ushort) /* */ -DEFINE_INTEGER_PARSERS(int) /* Don't use semicolons after these */ -DEFINE_INTEGER_PARSERS(uint) /* statements because they can cause */ -DEFINE_INTEGER_PARSERS(long) /* compiler warnings if the checking */ -DEFINE_INTEGER_PARSERS(ulong) /* level is turned up high enough. */ -DEFINE_INTEGER_PARSERS(longlong) /* */ -DEFINE_INTEGER_PARSERS(ulonglong) /* */ - -#undef DEFINE_INTEGER_PARSERS - -} // namespace pcrecpp + if (options_.utf8()) { + while (matchend < static_cast<int>(str->length()) && + ((*str)[matchend] & 0xc0) == 0x80) + matchend++; + } +#endif + if (start < static_cast<int>(str->length())) + out.append(*str, start, matchend - start); + start = matchend; + last_match_was_empty_string = false; + continue; + } + } else { + matches = TryMatch(*str, start, UNANCHORED, true, vec, kVecSize); + if (matches <= 0) + break; + } + int matchstart = vec[0], matchend = vec[1]; + assert(matchstart >= start); + assert(matchend >= matchstart); + out.append(*str, start, matchstart - start); + Rewrite(&out, rewrite, *str, vec, matches); + start = matchend; + count++; + last_match_was_empty_string = (matchstart == matchend); + } + + if (count == 0) + return 0; + + if (start < static_cast<int>(str->length())) + out.append(*str, start, str->length() - start); + swap(out, *str); + return count; +} + +bool RE::Extract(const StringPiece& rewrite, + const StringPiece& text, + string *out) const { + int vec[kVecSize]; + int matches = TryMatch(text, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + out->erase(); + return Rewrite(out, rewrite, text, vec, matches); +} + +/*static*/ string RE::QuoteMeta(const StringPiece& unquoted) { + string result; + + // Escape any ascii character not in [A-Za-z_0-9]. + // + // Note that it's legal to escape a character even if it has no + // special meaning in a regular expression -- so this function does + // that. (This also makes it identical to the perl function of the + // same name; see `perldoc -f quotemeta`.) The one exception is + // escaping NUL: rather than doing backslash + NUL, like perl does, + // we do '\0', because pcre itself doesn't take embedded NUL chars. + for (int ii = 0; ii < unquoted.size(); ++ii) { + // Note that using 'isalnum' here raises the benchmark time from + // 32ns to 58ns: + if (unquoted[ii] == '\0') { + result += "\\0"; + } else if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && + (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && + (unquoted[ii] < '0' || unquoted[ii] > '9') && + unquoted[ii] != '_' && + // If this is the part of a UTF8 or Latin1 character, we need + // to copy this byte without escaping. Experimentally this is + // what works correctly with the regexp library. + !(unquoted[ii] & 128)) { + result += '\\'; + result += unquoted[ii]; + } else { + result += unquoted[ii]; + } + } + + return result; +} + +/***** Actual matching and rewriting code *****/ + +int RE::TryMatch(const StringPiece& text, + int startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const { + pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_; + if (re == NULL) { + //fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str()); + return 0; + } + + pcre_extra extra = { 0, 0, 0, 0, 0, 0, 0, 0 }; + if (options_.match_limit() > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT; + extra.match_limit = options_.match_limit(); + } + if (options_.match_limit_recursion() > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + extra.match_limit_recursion = options_.match_limit_recursion(); + } + + // int options = 0; + // Changed by PH as a result of bugzilla #1288 + int options = (options_.all_options() & PCRE_NO_UTF8_CHECK); + + if (anchor != UNANCHORED) + options |= PCRE_ANCHORED; + if (!empty_ok) + options |= PCRE_NOTEMPTY; + + int rc = pcre_exec(re, // The regular expression object + &extra, + (text.data() == NULL) ? "" : text.data(), + text.size(), + startpos, + options, + vec, + vecsize); + + // Handle errors + if (rc == PCRE_ERROR_NOMATCH) { + return 0; + } else if (rc < 0) { + //fprintf(stderr, "Unexpected return code: %d when matching '%s'\n", + // re, pattern_.c_str()); + return 0; + } else if (rc == 0) { + // pcre_exec() returns 0 as a special case when the number of + // capturing subpatterns exceeds the size of the vector. + // When this happens, there is a match and the output vector + // is filled, but we miss out on the positions of the extra subpatterns. + rc = vecsize / 2; + } + + return rc; +} + +bool RE::DoMatchImpl(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const* args, + int n, + int* vec, + int vecsize) const { + assert((1 + n) * 3 <= vecsize); // results + PCRE workspace + int matches = TryMatch(text, 0, anchor, true, vec, vecsize); + assert(matches >= 0); // TryMatch never returns negatives + if (matches == 0) + return false; + + *consumed = vec[1]; + + if (n == 0 || args == NULL) { + // We are not interested in results + return true; + } + + if (NumberOfCapturingGroups() < n) { + // RE has fewer capturing groups than number of arg pointers passed in + return false; + } + + // If we got here, we must have matched the whole pattern. + // We do not need (can not do) any more checks on the value of 'matches' here + // -- see the comment for TryMatch. + for (int i = 0; i < n; i++) { + const int start = vec[2*(i+1)]; + const int limit = vec[2*(i+1)+1]; + if (!args[i]->Parse(text.data() + start, limit-start)) { + // TODO: Should we indicate what the error was? + return false; + } + } + + return true; +} + +bool RE::DoMatch(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const args[], + int n) const { + assert(n >= 0); + size_t const vecsize = (1 + n) * 3; // results + PCRE workspace + // (as for kVecSize) + int space[21]; // use stack allocation for small vecsize (common case) + int* vec = vecsize <= 21 ? space : new int[vecsize]; + bool retval = DoMatchImpl(text, anchor, consumed, args, n, vec, (int)vecsize); + if (vec != space) delete [] vec; + return retval; +} + +bool RE::Rewrite(string *out, const StringPiece &rewrite, + const StringPiece &text, int *vec, int veclen) const { + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + int c = *s; + if (c == '\\') { + c = *++s; + if (isdigit(c)) { + int n = (c - '0'); + if (n >= veclen) { + //fprintf(stderr, requested group %d in regexp %.*s\n", + // n, rewrite.size(), rewrite.data()); + return false; + } + int start = vec[2 * n]; + if (start >= 0) + out->append(text.data() + start, vec[2 * n + 1] - start); + } else if (c == '\\') { + *out += '\\'; + } else { + //fprintf(stderr, "invalid rewrite pattern: %.*s\n", + // rewrite.size(), rewrite.data()); + return false; + } + } else { + *out += c; + } + } + return true; +} + +// Return the number of capturing subpatterns, or -1 if the +// regexp wasn't valid on construction. +int RE::NumberOfCapturingGroups() const { + if (re_partial_ == NULL) return -1; + + int result; + int pcre_retval = pcre_fullinfo(re_partial_, // The regular expression object + NULL, // We did not study the pattern + PCRE_INFO_CAPTURECOUNT, + &result); + assert(pcre_retval == 0); + return result; +} + +/***** Parsers for various types *****/ + +bool Arg::parse_null(const char* str, int n, void* dest) { + (void)str; + (void)n; + // We fail if somebody asked us to store into a non-NULL void* pointer + return (dest == NULL); +} + +bool Arg::parse_string(const char* str, int n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast<string*>(dest)->assign(str, n); + return true; +} + +bool Arg::parse_stringpiece(const char* str, int n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast<StringPiece*>(dest)->set(str, n); + return true; +} + +bool Arg::parse_char(const char* str, int n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast<char*>(dest)) = str[0]; + return true; +} + +bool Arg::parse_uchar(const char* str, int n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast<unsigned char*>(dest)) = str[0]; + return true; +} + +// Largest number spec that we are willing to parse +static const int kMaxNumberLength = 32; + +// REQUIRES "buf" must have length at least kMaxNumberLength+1 +// REQUIRES "n > 0" +// Copies "str" into "buf" and null-terminates if necessary. +// Returns one of: +// a. "str" if no termination is needed +// b. "buf" if the string was copied and null-terminated +// c. "" if the input was invalid and has no hope of being parsed +static const char* TerminateNumber(char* buf, const char* str, int n) { + if ((n > 0) && isspace(*str)) { + // We are less forgiving than the strtoxxx() routines and do not + // allow leading spaces. + return ""; + } + + // See if the character right after the input text may potentially + // look like a digit. + if (isdigit(str[n]) || + ((str[n] >= 'a') && (str[n] <= 'f')) || + ((str[n] >= 'A') && (str[n] <= 'F'))) { + if (n > kMaxNumberLength) return ""; // Input too big to be a valid number + memcpy(buf, str, n); + buf[n] = '\0'; + return buf; + } else { + // We can parse right out of the supplied string, so return it. + return str; + } +} + +bool Arg::parse_long_radix(const char* str, + int n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; + long r = strtol(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast<long*>(dest)) = r; + return true; +} + +bool Arg::parse_ulong_radix(const char* str, + int n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') return false; // strtoul() on a negative number?! + char* end; + errno = 0; + unsigned long r = strtoul(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast<unsigned long*>(dest)) = r; + return true; +} + +bool Arg::parse_short_radix(const char* str, + int n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if (r < SHRT_MIN || r > SHRT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast<short*>(dest)) = static_cast<short>(r); + return true; +} + +bool Arg::parse_ushort_radix(const char* str, + int n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if (r > USHRT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast<unsigned short*>(dest)) = static_cast<unsigned short>(r); + return true; +} + +bool Arg::parse_int_radix(const char* str, + int n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if (r < INT_MIN || r > INT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast<int*>(dest)) = r; + return true; +} + +bool Arg::parse_uint_radix(const char* str, + int n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if (r > UINT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast<unsigned int*>(dest)) = r; + return true; +} + +bool Arg::parse_longlong_radix(const char* str, + int n, + void* dest, + int radix) { +#ifndef HAVE_LONG_LONG + return false; +#else + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; +#if defined HAVE_STRTOQ + long long r = strtoq(str, &end, radix); +#elif defined HAVE_STRTOLL + long long r = strtoll(str, &end, radix); +#elif defined HAVE__STRTOI64 + long long r = _strtoi64(str, &end, radix); +#elif defined HAVE_STRTOIMAX + long long r = strtoimax(str, &end, radix); +#else +#error parse_longlong_radix: cannot convert input to a long-long +#endif + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast<long long*>(dest)) = r; + return true; +#endif /* HAVE_LONG_LONG */ +} + +bool Arg::parse_ulonglong_radix(const char* str, + int n, + void* dest, + int radix) { +#ifndef HAVE_UNSIGNED_LONG_LONG + return false; +#else + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') return false; // strtoull() on a negative number?! + char* end; + errno = 0; +#if defined HAVE_STRTOQ + unsigned long long r = strtouq(str, &end, radix); +#elif defined HAVE_STRTOLL + unsigned long long r = strtoull(str, &end, radix); +#elif defined HAVE__STRTOI64 + unsigned long long r = _strtoui64(str, &end, radix); +#elif defined HAVE_STRTOIMAX + unsigned long long r = strtoumax(str, &end, radix); +#else +#error parse_ulonglong_radix: cannot convert input to a long-long +#endif + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast<unsigned long long*>(dest)) = r; + return true; +#endif /* HAVE_UNSIGNED_LONG_LONG */ +} + +bool Arg::parse_double(const char* str, int n, void* dest) { + if (n == 0) return false; + static const int kMaxLength = 200; + char buf[kMaxLength]; + if (n >= kMaxLength) return false; + memcpy(buf, str, n); + buf[n] = '\0'; + errno = 0; + char* end; + double r = strtod(buf, &end); + if (end != buf + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast<double*>(dest)) = r; + return true; +} + +bool Arg::parse_float(const char* str, int n, void* dest) { + double r; + if (!parse_double(str, n, &r)) return false; + if (dest == NULL) return true; + *(reinterpret_cast<float*>(dest)) = static_cast<float>(r); + return true; +} + + +#define DEFINE_INTEGER_PARSERS(name) \ + bool Arg::parse_##name(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 10); \ + } \ + bool Arg::parse_##name##_hex(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 16); \ + } \ + bool Arg::parse_##name##_octal(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 8); \ + } \ + bool Arg::parse_##name##_cradix(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 0); \ + } + +DEFINE_INTEGER_PARSERS(short) /* */ +DEFINE_INTEGER_PARSERS(ushort) /* */ +DEFINE_INTEGER_PARSERS(int) /* Don't use semicolons after these */ +DEFINE_INTEGER_PARSERS(uint) /* statements because they can cause */ +DEFINE_INTEGER_PARSERS(long) /* compiler warnings if the checking */ +DEFINE_INTEGER_PARSERS(ulong) /* level is turned up high enough. */ +DEFINE_INTEGER_PARSERS(longlong) /* */ +DEFINE_INTEGER_PARSERS(ulonglong) /* */ + +#undef DEFINE_INTEGER_PARSERS + +} // namespace pcrecpp diff --git a/contrib/libs/pcre/pcrecpp.h b/contrib/libs/pcre/pcrecpp.h index dffe9a34ad..e57066b545 100644 --- a/contrib/libs/pcre/pcrecpp.h +++ b/contrib/libs/pcre/pcrecpp.h @@ -1,710 +1,710 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// Support for PCRE_XXX modifiers added by Giuseppe Maxia, July 2005 - -#ifndef _PCRECPP_H -#define _PCRECPP_H - -// C++ interface to the pcre regular-expression library. RE supports -// Perl-style regular expressions (with extensions like \d, \w, \s, -// ...). -// -// ----------------------------------------------------------------------- -// REGEXP SYNTAX: -// -// This module is part of the pcre library and hence supports its syntax -// for regular expressions. -// -// The syntax is pretty similar to Perl's. For those not familiar -// with Perl's regular expressions, here are some examples of the most -// commonly used extensions: -// -// "hello (\\w+) world" -- \w matches a "word" character -// "version (\\d+)" -- \d matches a digit -// "hello\\s+world" -- \s matches any whitespace character -// "\\b(\\w+)\\b" -- \b matches empty string at a word boundary -// "(?i)hello" -- (?i) turns on case-insensitive matching -// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible -// -// ----------------------------------------------------------------------- -// MATCHING INTERFACE: -// -// The "FullMatch" operation checks that supplied text matches a -// supplied pattern exactly. -// -// Example: successful match -// pcrecpp::RE re("h.*o"); -// re.FullMatch("hello"); -// -// Example: unsuccessful match (requires full match): -// pcrecpp::RE re("e"); -// !re.FullMatch("hello"); -// -// Example: creating a temporary RE object: -// pcrecpp::RE("h.*o").FullMatch("hello"); -// -// You can pass in a "const char*" or a "string" for "text". The -// examples below tend to use a const char*. -// -// You can, as in the different examples above, store the RE object -// explicitly in a variable or use a temporary RE object. The -// examples below use one mode or the other arbitrarily. Either -// could correctly be used for any of these examples. -// -// ----------------------------------------------------------------------- -// MATCHING WITH SUB-STRING EXTRACTION: -// -// You can supply extra pointer arguments to extract matched subpieces. -// -// Example: extracts "ruby" into "s" and 1234 into "i" -// int i; -// string s; -// pcrecpp::RE re("(\\w+):(\\d+)"); -// re.FullMatch("ruby:1234", &s, &i); -// -// Example: does not try to extract any extra sub-patterns -// re.FullMatch("ruby:1234", &s); -// -// Example: does not try to extract into NULL -// re.FullMatch("ruby:1234", NULL, &i); -// -// Example: integer overflow causes failure -// !re.FullMatch("ruby:1234567891234", NULL, &i); -// -// Example: fails because there aren't enough sub-patterns: -// !pcrecpp::RE("\\w+:\\d+").FullMatch("ruby:1234", &s); -// -// Example: fails because string cannot be stored in integer -// !pcrecpp::RE("(.*)").FullMatch("ruby", &i); -// -// The provided pointer arguments can be pointers to any scalar numeric -// type, or one of -// string (matched piece is copied to string) -// StringPiece (StringPiece is mutated to point to matched piece) -// T (where "bool T::ParseFrom(const char*, int)" exists) -// NULL (the corresponding matched sub-pattern is not copied) -// -// CAVEAT: An optional sub-pattern that does not exist in the matched -// string is assigned the empty string. Therefore, the following will -// return false (because the empty string is not a valid number): -// int number; -// pcrecpp::RE::FullMatch("abc", "[a-z]+(\\d+)?", &number); -// -// ----------------------------------------------------------------------- -// DO_MATCH -// -// The matching interface supports at most 16 arguments per call. -// If you need more, consider using the more general interface -// pcrecpp::RE::DoMatch(). See pcrecpp.h for the signature for DoMatch. -// -// ----------------------------------------------------------------------- -// PARTIAL MATCHES -// -// You can use the "PartialMatch" operation when you want the pattern -// to match any substring of the text. -// -// Example: simple search for a string: -// pcrecpp::RE("ell").PartialMatch("hello"); -// -// Example: find first number in a string: -// int number; -// pcrecpp::RE re("(\\d+)"); -// re.PartialMatch("x*100 + 20", &number); -// assert(number == 100); -// -// ----------------------------------------------------------------------- -// UTF-8 AND THE MATCHING INTERFACE: -// -// By default, pattern and text are plain text, one byte per character. -// The UTF8 flag, passed to the constructor, causes both pattern -// and string to be treated as UTF-8 text, still a byte stream but -// potentially multiple bytes per character. In practice, the text -// is likelier to be UTF-8 than the pattern, but the match returned -// may depend on the UTF8 flag, so always use it when matching -// UTF8 text. E.g., "." will match one byte normally but with UTF8 -// set may match up to three bytes of a multi-byte character. -// -// Example: -// pcrecpp::RE_Options options; -// options.set_utf8(); -// pcrecpp::RE re(utf8_pattern, options); -// re.FullMatch(utf8_string); -// -// Example: using the convenience function UTF8(): -// pcrecpp::RE re(utf8_pattern, pcrecpp::UTF8()); -// re.FullMatch(utf8_string); -// -// NOTE: The UTF8 option is ignored if pcre was not configured with the -// --enable-utf8 flag. -// -// ----------------------------------------------------------------------- -// PASSING MODIFIERS TO THE REGULAR EXPRESSION ENGINE -// -// PCRE defines some modifiers to change the behavior of the regular -// expression engine. -// The C++ wrapper defines an auxiliary class, RE_Options, as a vehicle -// to pass such modifiers to a RE class. -// -// Currently, the following modifiers are supported -// -// modifier description Perl corresponding -// -// PCRE_CASELESS case insensitive match /i -// PCRE_MULTILINE multiple lines match /m -// PCRE_DOTALL dot matches newlines /s -// PCRE_DOLLAR_ENDONLY $ matches only at end N/A -// PCRE_EXTRA strict escape parsing N/A -// PCRE_EXTENDED ignore whitespaces /x -// PCRE_UTF8 handles UTF8 chars built-in -// PCRE_UNGREEDY reverses * and *? N/A -// PCRE_NO_AUTO_CAPTURE disables matching parens N/A (*) -// -// (For a full account on how each modifier works, please check the -// PCRE API reference manual). -// -// (*) Both Perl and PCRE allow non matching parentheses by means of the -// "?:" modifier within the pattern itself. e.g. (?:ab|cd) does not -// capture, while (ab|cd) does. -// -// For each modifier, there are two member functions whose name is made -// out of the modifier in lowercase, without the "PCRE_" prefix. For -// instance, PCRE_CASELESS is handled by -// bool caseless(), -// which returns true if the modifier is set, and -// RE_Options & set_caseless(bool), -// which sets or unsets the modifier. -// -// Moreover, PCRE_EXTRA_MATCH_LIMIT can be accessed through the -// set_match_limit() and match_limit() member functions. -// Setting match_limit to a non-zero value will limit the executation of -// pcre to keep it from doing bad things like blowing the stack or taking -// an eternity to return a result. A value of 5000 is good enough to stop -// stack blowup in a 2MB thread stack. Setting match_limit to zero will -// disable match limiting. Alternately, you can set match_limit_recursion() -// which uses PCRE_EXTRA_MATCH_LIMIT_RECURSION to limit how much pcre -// recurses. match_limit() caps the number of matches pcre does; -// match_limit_recrusion() caps the depth of recursion. -// -// Normally, to pass one or more modifiers to a RE class, you declare -// a RE_Options object, set the appropriate options, and pass this -// object to a RE constructor. Example: -// -// RE_options opt; -// opt.set_caseless(true); -// -// if (RE("HELLO", opt).PartialMatch("hello world")) ... -// -// RE_options has two constructors. The default constructor takes no -// arguments and creates a set of flags that are off by default. -// -// The optional parameter 'option_flags' is to facilitate transfer -// of legacy code from C programs. This lets you do -// RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); -// -// But new code is better off doing -// RE(pattern, -// RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); -// (See below) -// -// If you are going to pass one of the most used modifiers, there are some -// convenience functions that return a RE_Options class with the -// appropriate modifier already set: -// CASELESS(), UTF8(), MULTILINE(), DOTALL(), EXTENDED() -// -// If you need to set several options at once, and you don't want to go -// through the pains of declaring a RE_Options object and setting several -// options, there is a parallel method that give you such ability on the -// fly. You can concatenate several set_xxxxx member functions, since each -// of them returns a reference to its class object. e.g.: to pass -// PCRE_CASELESS, PCRE_EXTENDED, and PCRE_MULTILINE to a RE with one -// statement, you may write -// -// RE(" ^ xyz \\s+ .* blah$", RE_Options() -// .set_caseless(true) -// .set_extended(true) -// .set_multiline(true)).PartialMatch(sometext); -// -// ----------------------------------------------------------------------- -// SCANNING TEXT INCREMENTALLY -// -// The "Consume" operation may be useful if you want to repeatedly -// match regular expressions at the front of a string and skip over -// them as they match. This requires use of the "StringPiece" type, -// which represents a sub-range of a real string. Like RE, StringPiece -// is defined in the pcrecpp namespace. -// -// Example: read lines of the form "var = value" from a string. -// string contents = ...; // Fill string somehow -// pcrecpp::StringPiece input(contents); // Wrap in a StringPiece -// -// string var; -// int value; -// pcrecpp::RE re("(\\w+) = (\\d+)\n"); -// while (re.Consume(&input, &var, &value)) { -// ...; -// } -// -// Each successful call to "Consume" will set "var/value", and also -// advance "input" so it points past the matched text. -// -// The "FindAndConsume" operation is similar to "Consume" but does not -// anchor your match at the beginning of the string. For example, you -// could extract all words from a string by repeatedly calling -// pcrecpp::RE("(\\w+)").FindAndConsume(&input, &word) -// -// ----------------------------------------------------------------------- -// PARSING HEX/OCTAL/C-RADIX NUMBERS -// -// By default, if you pass a pointer to a numeric value, the -// corresponding text is interpreted as a base-10 number. You can -// instead wrap the pointer with a call to one of the operators Hex(), -// Octal(), or CRadix() to interpret the text in another base. The -// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) -// prefixes, but defaults to base-10. -// -// Example: -// int a, b, c, d; -// pcrecpp::RE re("(.*) (.*) (.*) (.*)"); -// re.FullMatch("100 40 0100 0x40", -// pcrecpp::Octal(&a), pcrecpp::Hex(&b), -// pcrecpp::CRadix(&c), pcrecpp::CRadix(&d)); -// will leave 64 in a, b, c, and d. -// -// ----------------------------------------------------------------------- -// REPLACING PARTS OF STRINGS -// -// You can replace the first match of "pattern" in "str" with -// "rewrite". Within "rewrite", backslash-escaped digits (\1 to \9) -// can be used to insert text matching corresponding parenthesized -// group from the pattern. \0 in "rewrite" refers to the entire -// matching text. E.g., -// -// string s = "yabba dabba doo"; -// pcrecpp::RE("b+").Replace("d", &s); -// -// will leave "s" containing "yada dabba doo". The result is true if -// the pattern matches and a replacement occurs, or false otherwise. -// -// GlobalReplace() is like Replace(), except that it replaces all -// occurrences of the pattern in the string with the rewrite. -// Replacements are not subject to re-matching. E.g., -// -// string s = "yabba dabba doo"; -// pcrecpp::RE("b+").GlobalReplace("d", &s); -// -// will leave "s" containing "yada dada doo". It returns the number -// of replacements made. -// -// Extract() is like Replace(), except that if the pattern matches, -// "rewrite" is copied into "out" (an additional argument) with -// substitutions. The non-matching portions of "text" are ignored. -// Returns true iff a match occurred and the extraction happened -// successfully. If no match occurs, the string is left unaffected. - - -#include <string> +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// Support for PCRE_XXX modifiers added by Giuseppe Maxia, July 2005 + +#ifndef _PCRECPP_H +#define _PCRECPP_H + +// C++ interface to the pcre regular-expression library. RE supports +// Perl-style regular expressions (with extensions like \d, \w, \s, +// ...). +// +// ----------------------------------------------------------------------- +// REGEXP SYNTAX: +// +// This module is part of the pcre library and hence supports its syntax +// for regular expressions. +// +// The syntax is pretty similar to Perl's. For those not familiar +// with Perl's regular expressions, here are some examples of the most +// commonly used extensions: +// +// "hello (\\w+) world" -- \w matches a "word" character +// "version (\\d+)" -- \d matches a digit +// "hello\\s+world" -- \s matches any whitespace character +// "\\b(\\w+)\\b" -- \b matches empty string at a word boundary +// "(?i)hello" -- (?i) turns on case-insensitive matching +// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible +// +// ----------------------------------------------------------------------- +// MATCHING INTERFACE: +// +// The "FullMatch" operation checks that supplied text matches a +// supplied pattern exactly. +// +// Example: successful match +// pcrecpp::RE re("h.*o"); +// re.FullMatch("hello"); +// +// Example: unsuccessful match (requires full match): +// pcrecpp::RE re("e"); +// !re.FullMatch("hello"); +// +// Example: creating a temporary RE object: +// pcrecpp::RE("h.*o").FullMatch("hello"); +// +// You can pass in a "const char*" or a "string" for "text". The +// examples below tend to use a const char*. +// +// You can, as in the different examples above, store the RE object +// explicitly in a variable or use a temporary RE object. The +// examples below use one mode or the other arbitrarily. Either +// could correctly be used for any of these examples. +// +// ----------------------------------------------------------------------- +// MATCHING WITH SUB-STRING EXTRACTION: +// +// You can supply extra pointer arguments to extract matched subpieces. +// +// Example: extracts "ruby" into "s" and 1234 into "i" +// int i; +// string s; +// pcrecpp::RE re("(\\w+):(\\d+)"); +// re.FullMatch("ruby:1234", &s, &i); +// +// Example: does not try to extract any extra sub-patterns +// re.FullMatch("ruby:1234", &s); +// +// Example: does not try to extract into NULL +// re.FullMatch("ruby:1234", NULL, &i); +// +// Example: integer overflow causes failure +// !re.FullMatch("ruby:1234567891234", NULL, &i); +// +// Example: fails because there aren't enough sub-patterns: +// !pcrecpp::RE("\\w+:\\d+").FullMatch("ruby:1234", &s); +// +// Example: fails because string cannot be stored in integer +// !pcrecpp::RE("(.*)").FullMatch("ruby", &i); +// +// The provided pointer arguments can be pointers to any scalar numeric +// type, or one of +// string (matched piece is copied to string) +// StringPiece (StringPiece is mutated to point to matched piece) +// T (where "bool T::ParseFrom(const char*, int)" exists) +// NULL (the corresponding matched sub-pattern is not copied) +// +// CAVEAT: An optional sub-pattern that does not exist in the matched +// string is assigned the empty string. Therefore, the following will +// return false (because the empty string is not a valid number): +// int number; +// pcrecpp::RE::FullMatch("abc", "[a-z]+(\\d+)?", &number); +// +// ----------------------------------------------------------------------- +// DO_MATCH +// +// The matching interface supports at most 16 arguments per call. +// If you need more, consider using the more general interface +// pcrecpp::RE::DoMatch(). See pcrecpp.h for the signature for DoMatch. +// +// ----------------------------------------------------------------------- +// PARTIAL MATCHES +// +// You can use the "PartialMatch" operation when you want the pattern +// to match any substring of the text. +// +// Example: simple search for a string: +// pcrecpp::RE("ell").PartialMatch("hello"); +// +// Example: find first number in a string: +// int number; +// pcrecpp::RE re("(\\d+)"); +// re.PartialMatch("x*100 + 20", &number); +// assert(number == 100); +// +// ----------------------------------------------------------------------- +// UTF-8 AND THE MATCHING INTERFACE: +// +// By default, pattern and text are plain text, one byte per character. +// The UTF8 flag, passed to the constructor, causes both pattern +// and string to be treated as UTF-8 text, still a byte stream but +// potentially multiple bytes per character. In practice, the text +// is likelier to be UTF-8 than the pattern, but the match returned +// may depend on the UTF8 flag, so always use it when matching +// UTF8 text. E.g., "." will match one byte normally but with UTF8 +// set may match up to three bytes of a multi-byte character. +// +// Example: +// pcrecpp::RE_Options options; +// options.set_utf8(); +// pcrecpp::RE re(utf8_pattern, options); +// re.FullMatch(utf8_string); +// +// Example: using the convenience function UTF8(): +// pcrecpp::RE re(utf8_pattern, pcrecpp::UTF8()); +// re.FullMatch(utf8_string); +// +// NOTE: The UTF8 option is ignored if pcre was not configured with the +// --enable-utf8 flag. +// +// ----------------------------------------------------------------------- +// PASSING MODIFIERS TO THE REGULAR EXPRESSION ENGINE +// +// PCRE defines some modifiers to change the behavior of the regular +// expression engine. +// The C++ wrapper defines an auxiliary class, RE_Options, as a vehicle +// to pass such modifiers to a RE class. +// +// Currently, the following modifiers are supported +// +// modifier description Perl corresponding +// +// PCRE_CASELESS case insensitive match /i +// PCRE_MULTILINE multiple lines match /m +// PCRE_DOTALL dot matches newlines /s +// PCRE_DOLLAR_ENDONLY $ matches only at end N/A +// PCRE_EXTRA strict escape parsing N/A +// PCRE_EXTENDED ignore whitespaces /x +// PCRE_UTF8 handles UTF8 chars built-in +// PCRE_UNGREEDY reverses * and *? N/A +// PCRE_NO_AUTO_CAPTURE disables matching parens N/A (*) +// +// (For a full account on how each modifier works, please check the +// PCRE API reference manual). +// +// (*) Both Perl and PCRE allow non matching parentheses by means of the +// "?:" modifier within the pattern itself. e.g. (?:ab|cd) does not +// capture, while (ab|cd) does. +// +// For each modifier, there are two member functions whose name is made +// out of the modifier in lowercase, without the "PCRE_" prefix. For +// instance, PCRE_CASELESS is handled by +// bool caseless(), +// which returns true if the modifier is set, and +// RE_Options & set_caseless(bool), +// which sets or unsets the modifier. +// +// Moreover, PCRE_EXTRA_MATCH_LIMIT can be accessed through the +// set_match_limit() and match_limit() member functions. +// Setting match_limit to a non-zero value will limit the executation of +// pcre to keep it from doing bad things like blowing the stack or taking +// an eternity to return a result. A value of 5000 is good enough to stop +// stack blowup in a 2MB thread stack. Setting match_limit to zero will +// disable match limiting. Alternately, you can set match_limit_recursion() +// which uses PCRE_EXTRA_MATCH_LIMIT_RECURSION to limit how much pcre +// recurses. match_limit() caps the number of matches pcre does; +// match_limit_recrusion() caps the depth of recursion. +// +// Normally, to pass one or more modifiers to a RE class, you declare +// a RE_Options object, set the appropriate options, and pass this +// object to a RE constructor. Example: +// +// RE_options opt; +// opt.set_caseless(true); +// +// if (RE("HELLO", opt).PartialMatch("hello world")) ... +// +// RE_options has two constructors. The default constructor takes no +// arguments and creates a set of flags that are off by default. +// +// The optional parameter 'option_flags' is to facilitate transfer +// of legacy code from C programs. This lets you do +// RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); +// +// But new code is better off doing +// RE(pattern, +// RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); +// (See below) +// +// If you are going to pass one of the most used modifiers, there are some +// convenience functions that return a RE_Options class with the +// appropriate modifier already set: +// CASELESS(), UTF8(), MULTILINE(), DOTALL(), EXTENDED() +// +// If you need to set several options at once, and you don't want to go +// through the pains of declaring a RE_Options object and setting several +// options, there is a parallel method that give you such ability on the +// fly. You can concatenate several set_xxxxx member functions, since each +// of them returns a reference to its class object. e.g.: to pass +// PCRE_CASELESS, PCRE_EXTENDED, and PCRE_MULTILINE to a RE with one +// statement, you may write +// +// RE(" ^ xyz \\s+ .* blah$", RE_Options() +// .set_caseless(true) +// .set_extended(true) +// .set_multiline(true)).PartialMatch(sometext); +// +// ----------------------------------------------------------------------- +// SCANNING TEXT INCREMENTALLY +// +// The "Consume" operation may be useful if you want to repeatedly +// match regular expressions at the front of a string and skip over +// them as they match. This requires use of the "StringPiece" type, +// which represents a sub-range of a real string. Like RE, StringPiece +// is defined in the pcrecpp namespace. +// +// Example: read lines of the form "var = value" from a string. +// string contents = ...; // Fill string somehow +// pcrecpp::StringPiece input(contents); // Wrap in a StringPiece +// +// string var; +// int value; +// pcrecpp::RE re("(\\w+) = (\\d+)\n"); +// while (re.Consume(&input, &var, &value)) { +// ...; +// } +// +// Each successful call to "Consume" will set "var/value", and also +// advance "input" so it points past the matched text. +// +// The "FindAndConsume" operation is similar to "Consume" but does not +// anchor your match at the beginning of the string. For example, you +// could extract all words from a string by repeatedly calling +// pcrecpp::RE("(\\w+)").FindAndConsume(&input, &word) +// +// ----------------------------------------------------------------------- +// PARSING HEX/OCTAL/C-RADIX NUMBERS +// +// By default, if you pass a pointer to a numeric value, the +// corresponding text is interpreted as a base-10 number. You can +// instead wrap the pointer with a call to one of the operators Hex(), +// Octal(), or CRadix() to interpret the text in another base. The +// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) +// prefixes, but defaults to base-10. +// +// Example: +// int a, b, c, d; +// pcrecpp::RE re("(.*) (.*) (.*) (.*)"); +// re.FullMatch("100 40 0100 0x40", +// pcrecpp::Octal(&a), pcrecpp::Hex(&b), +// pcrecpp::CRadix(&c), pcrecpp::CRadix(&d)); +// will leave 64 in a, b, c, and d. +// +// ----------------------------------------------------------------------- +// REPLACING PARTS OF STRINGS +// +// You can replace the first match of "pattern" in "str" with +// "rewrite". Within "rewrite", backslash-escaped digits (\1 to \9) +// can be used to insert text matching corresponding parenthesized +// group from the pattern. \0 in "rewrite" refers to the entire +// matching text. E.g., +// +// string s = "yabba dabba doo"; +// pcrecpp::RE("b+").Replace("d", &s); +// +// will leave "s" containing "yada dabba doo". The result is true if +// the pattern matches and a replacement occurs, or false otherwise. +// +// GlobalReplace() is like Replace(), except that it replaces all +// occurrences of the pattern in the string with the rewrite. +// Replacements are not subject to re-matching. E.g., +// +// string s = "yabba dabba doo"; +// pcrecpp::RE("b+").GlobalReplace("d", &s); +// +// will leave "s" containing "yada dada doo". It returns the number +// of replacements made. +// +// Extract() is like Replace(), except that if the pattern matches, +// "rewrite" is copied into "out" (an additional argument) with +// substitutions. The non-matching portions of "text" are ignored. +// Returns true iff a match occurred and the extraction happened +// successfully. If no match occurs, the string is left unaffected. + + +#include <string> #include "pcre.h" #include "pcrecpparg.h" // defines the Arg class -// This isn't technically needed here, but we include it -// anyway so folks who include pcrecpp.h don't have to. +// This isn't technically needed here, but we include it +// anyway so folks who include pcrecpp.h don't have to. #include "pcre_stringpiece.h" - -namespace pcrecpp { - -#define PCRE_SET_OR_CLEAR(b, o) \ - if (b) all_options_ |= (o); else all_options_ &= ~(o); \ - return *this - -#define PCRE_IS_SET(o) \ - (all_options_ & o) == o - -/***** Compiling regular expressions: the RE class *****/ - -// RE_Options allow you to set options to be passed along to pcre, -// along with other options we put on top of pcre. -// Only 9 modifiers, plus match_limit and match_limit_recursion, -// are supported now. -class PCRECPP_EXP_DEFN RE_Options { - public: - // constructor - RE_Options() : match_limit_(0), match_limit_recursion_(0), all_options_(0) {} - - // alternative constructor. - // To facilitate transfer of legacy code from C programs - // - // This lets you do - // RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); - // But new code is better off doing - // RE(pattern, - // RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); - RE_Options(int option_flags) : match_limit_(0), match_limit_recursion_(0), - all_options_(option_flags) {} - // we're fine with the default destructor, copy constructor, etc. - - // accessors and mutators - int match_limit() const { return match_limit_; }; - RE_Options &set_match_limit(int limit) { - match_limit_ = limit; - return *this; - } - - int match_limit_recursion() const { return match_limit_recursion_; }; - RE_Options &set_match_limit_recursion(int limit) { - match_limit_recursion_ = limit; - return *this; - } - - bool caseless() const { - return PCRE_IS_SET(PCRE_CASELESS); - } - RE_Options &set_caseless(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_CASELESS); - } - - bool multiline() const { - return PCRE_IS_SET(PCRE_MULTILINE); - } - RE_Options &set_multiline(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_MULTILINE); - } - - bool dotall() const { - return PCRE_IS_SET(PCRE_DOTALL); - } - RE_Options &set_dotall(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_DOTALL); - } - - bool extended() const { - return PCRE_IS_SET(PCRE_EXTENDED); - } - RE_Options &set_extended(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_EXTENDED); - } - - bool dollar_endonly() const { - return PCRE_IS_SET(PCRE_DOLLAR_ENDONLY); - } - RE_Options &set_dollar_endonly(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_DOLLAR_ENDONLY); - } - - bool extra() const { - return PCRE_IS_SET(PCRE_EXTRA); - } - RE_Options &set_extra(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_EXTRA); - } - - bool ungreedy() const { - return PCRE_IS_SET(PCRE_UNGREEDY); - } - RE_Options &set_ungreedy(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_UNGREEDY); - } - - bool utf8() const { - return PCRE_IS_SET(PCRE_UTF8); - } - RE_Options &set_utf8(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_UTF8); - } - - bool no_auto_capture() const { - return PCRE_IS_SET(PCRE_NO_AUTO_CAPTURE); - } - RE_Options &set_no_auto_capture(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_NO_AUTO_CAPTURE); - } - - RE_Options &set_all_options(int opt) { - all_options_ = opt; - return *this; - } - int all_options() const { - return all_options_ ; - } - - // TODO: add other pcre flags - - private: - int match_limit_; - int match_limit_recursion_; - int all_options_; -}; - -// These functions return some common RE_Options -static inline RE_Options UTF8() { - return RE_Options().set_utf8(true); -} - -static inline RE_Options CASELESS() { - return RE_Options().set_caseless(true); -} -static inline RE_Options MULTILINE() { - return RE_Options().set_multiline(true); -} - -static inline RE_Options DOTALL() { - return RE_Options().set_dotall(true); -} - -static inline RE_Options EXTENDED() { - return RE_Options().set_extended(true); -} - -// Interface for regular expression matching. Also corresponds to a -// pre-compiled regular expression. An "RE" object is safe for -// concurrent use by multiple threads. -class PCRECPP_EXP_DEFN RE { - public: - // We provide implicit conversions from strings so that users can - // pass in a string or a "const char*" wherever an "RE" is expected. - RE(const string& pat) { Init(pat, NULL); } - RE(const string& pat, const RE_Options& option) { Init(pat, &option); } - RE(const char* pat) { Init(pat, NULL); } - RE(const char* pat, const RE_Options& option) { Init(pat, &option); } - RE(const unsigned char* pat) { - Init(reinterpret_cast<const char*>(pat), NULL); - } - RE(const unsigned char* pat, const RE_Options& option) { - Init(reinterpret_cast<const char*>(pat), &option); - } - - // Copy constructor & assignment - note that these are expensive - // because they recompile the expression. - RE(const RE& re) { Init(re.pattern_, &re.options_); } - const RE& operator=(const RE& re) { - if (this != &re) { - Cleanup(); - - // This is the code that originally came from Google - // Init(re.pattern_.c_str(), &re.options_); - - // This is the replacement from Ari Pollak - Init(re.pattern_, &re.options_); - } - return *this; - } - - - ~RE(); - - // The string specification for this RE. E.g. - // RE re("ab*c?d+"); - // re.pattern(); // "ab*c?d+" - const string& pattern() const { return pattern_; } - - // If RE could not be created properly, returns an error string. - // Else returns the empty string. - const string& error() const { return *error_; } - - /***** The useful part: the matching interface *****/ - - // This is provided so one can do pattern.ReplaceAll() just as - // easily as ReplaceAll(pattern-text, ....) - - bool FullMatch(const StringPiece& text, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool PartialMatch(const StringPiece& text, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool Consume(StringPiece* input, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool FindAndConsume(StringPiece* input, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool Replace(const StringPiece& rewrite, - string *str) const; - - int GlobalReplace(const StringPiece& rewrite, - string *str) const; - - bool Extract(const StringPiece &rewrite, - const StringPiece &text, - string *out) const; - - // Escapes all potentially meaningful regexp characters in - // 'unquoted'. The returned string, used as a regular expression, - // will exactly match the original string. For example, - // 1.5-2.0? - // may become: - // 1\.5\-2\.0\? - // Note QuoteMeta behaves the same as perl's QuoteMeta function, - // *except* that it escapes the NUL character (\0) as backslash + 0, - // rather than backslash + NUL. - static string QuoteMeta(const StringPiece& unquoted); - - - /***** Generic matching interface *****/ - - // Type of match (TODO: Should be restructured as part of RE_Options) - enum Anchor { - UNANCHORED, // No anchoring - ANCHOR_START, // Anchor at start only - ANCHOR_BOTH // Anchor at start and end - }; - - // General matching routine. Stores the length of the match in - // "*consumed" if successful. - bool DoMatch(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const* args, int n) const; - - // Return the number of capturing subpatterns, or -1 if the - // regexp wasn't valid on construction. - int NumberOfCapturingGroups() const; - - // The default value for an argument, to indicate the end of the argument - // list. This must be used only in optional argument defaults. It should NOT - // be passed explicitly. Some people have tried to use it like this: - // - // FullMatch(x, y, &z, no_arg, &w); - // - // This is a mistake, and will not work. - static Arg no_arg; - - private: - - void Init(const string& pattern, const RE_Options* options); - void Cleanup(); - - // Match against "text", filling in "vec" (up to "vecsize" * 2/3) with - // pairs of integers for the beginning and end positions of matched - // text. The first pair corresponds to the entire matched text; - // subsequent pairs correspond, in order, to parentheses-captured - // matches. Returns the number of pairs (one more than the number of - // the last subpattern with a match) if matching was successful - // and zero if the match failed. - // I.e. for RE("(foo)|(bar)|(baz)") it will return 2, 3, and 4 when matching - // against "foo", "bar", and "baz" respectively. - // When matching RE("(foo)|hello") against "hello", it will return 1. - // But the values for all subpattern are filled in into "vec". - int TryMatch(const StringPiece& text, - int startpos, - Anchor anchor, - bool empty_ok, - int *vec, - int vecsize) const; - - // Append the "rewrite" string, with backslash subsitutions from "text" - // and "vec", to string "out". - bool Rewrite(string *out, - const StringPiece& rewrite, - const StringPiece& text, - int *vec, - int veclen) const; - - // internal implementation for DoMatch - bool DoMatchImpl(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const args[], - int n, - int* vec, - int vecsize) const; - - // Compile the regexp for the specified anchoring mode - pcre* Compile(Anchor anchor); - - string pattern_; - RE_Options options_; - pcre* re_full_; // For full matches - pcre* re_partial_; // For partial matches - const string* error_; // Error indicator (or points to empty string) -}; - -} // namespace pcrecpp - -#endif /* _PCRECPP_H */ + +namespace pcrecpp { + +#define PCRE_SET_OR_CLEAR(b, o) \ + if (b) all_options_ |= (o); else all_options_ &= ~(o); \ + return *this + +#define PCRE_IS_SET(o) \ + (all_options_ & o) == o + +/***** Compiling regular expressions: the RE class *****/ + +// RE_Options allow you to set options to be passed along to pcre, +// along with other options we put on top of pcre. +// Only 9 modifiers, plus match_limit and match_limit_recursion, +// are supported now. +class PCRECPP_EXP_DEFN RE_Options { + public: + // constructor + RE_Options() : match_limit_(0), match_limit_recursion_(0), all_options_(0) {} + + // alternative constructor. + // To facilitate transfer of legacy code from C programs + // + // This lets you do + // RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); + // But new code is better off doing + // RE(pattern, + // RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); + RE_Options(int option_flags) : match_limit_(0), match_limit_recursion_(0), + all_options_(option_flags) {} + // we're fine with the default destructor, copy constructor, etc. + + // accessors and mutators + int match_limit() const { return match_limit_; }; + RE_Options &set_match_limit(int limit) { + match_limit_ = limit; + return *this; + } + + int match_limit_recursion() const { return match_limit_recursion_; }; + RE_Options &set_match_limit_recursion(int limit) { + match_limit_recursion_ = limit; + return *this; + } + + bool caseless() const { + return PCRE_IS_SET(PCRE_CASELESS); + } + RE_Options &set_caseless(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_CASELESS); + } + + bool multiline() const { + return PCRE_IS_SET(PCRE_MULTILINE); + } + RE_Options &set_multiline(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_MULTILINE); + } + + bool dotall() const { + return PCRE_IS_SET(PCRE_DOTALL); + } + RE_Options &set_dotall(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_DOTALL); + } + + bool extended() const { + return PCRE_IS_SET(PCRE_EXTENDED); + } + RE_Options &set_extended(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_EXTENDED); + } + + bool dollar_endonly() const { + return PCRE_IS_SET(PCRE_DOLLAR_ENDONLY); + } + RE_Options &set_dollar_endonly(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_DOLLAR_ENDONLY); + } + + bool extra() const { + return PCRE_IS_SET(PCRE_EXTRA); + } + RE_Options &set_extra(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_EXTRA); + } + + bool ungreedy() const { + return PCRE_IS_SET(PCRE_UNGREEDY); + } + RE_Options &set_ungreedy(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_UNGREEDY); + } + + bool utf8() const { + return PCRE_IS_SET(PCRE_UTF8); + } + RE_Options &set_utf8(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_UTF8); + } + + bool no_auto_capture() const { + return PCRE_IS_SET(PCRE_NO_AUTO_CAPTURE); + } + RE_Options &set_no_auto_capture(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_NO_AUTO_CAPTURE); + } + + RE_Options &set_all_options(int opt) { + all_options_ = opt; + return *this; + } + int all_options() const { + return all_options_ ; + } + + // TODO: add other pcre flags + + private: + int match_limit_; + int match_limit_recursion_; + int all_options_; +}; + +// These functions return some common RE_Options +static inline RE_Options UTF8() { + return RE_Options().set_utf8(true); +} + +static inline RE_Options CASELESS() { + return RE_Options().set_caseless(true); +} +static inline RE_Options MULTILINE() { + return RE_Options().set_multiline(true); +} + +static inline RE_Options DOTALL() { + return RE_Options().set_dotall(true); +} + +static inline RE_Options EXTENDED() { + return RE_Options().set_extended(true); +} + +// Interface for regular expression matching. Also corresponds to a +// pre-compiled regular expression. An "RE" object is safe for +// concurrent use by multiple threads. +class PCRECPP_EXP_DEFN RE { + public: + // We provide implicit conversions from strings so that users can + // pass in a string or a "const char*" wherever an "RE" is expected. + RE(const string& pat) { Init(pat, NULL); } + RE(const string& pat, const RE_Options& option) { Init(pat, &option); } + RE(const char* pat) { Init(pat, NULL); } + RE(const char* pat, const RE_Options& option) { Init(pat, &option); } + RE(const unsigned char* pat) { + Init(reinterpret_cast<const char*>(pat), NULL); + } + RE(const unsigned char* pat, const RE_Options& option) { + Init(reinterpret_cast<const char*>(pat), &option); + } + + // Copy constructor & assignment - note that these are expensive + // because they recompile the expression. + RE(const RE& re) { Init(re.pattern_, &re.options_); } + const RE& operator=(const RE& re) { + if (this != &re) { + Cleanup(); + + // This is the code that originally came from Google + // Init(re.pattern_.c_str(), &re.options_); + + // This is the replacement from Ari Pollak + Init(re.pattern_, &re.options_); + } + return *this; + } + + + ~RE(); + + // The string specification for this RE. E.g. + // RE re("ab*c?d+"); + // re.pattern(); // "ab*c?d+" + const string& pattern() const { return pattern_; } + + // If RE could not be created properly, returns an error string. + // Else returns the empty string. + const string& error() const { return *error_; } + + /***** The useful part: the matching interface *****/ + + // This is provided so one can do pattern.ReplaceAll() just as + // easily as ReplaceAll(pattern-text, ....) + + bool FullMatch(const StringPiece& text, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool PartialMatch(const StringPiece& text, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool Consume(StringPiece* input, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool FindAndConsume(StringPiece* input, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool Replace(const StringPiece& rewrite, + string *str) const; + + int GlobalReplace(const StringPiece& rewrite, + string *str) const; + + bool Extract(const StringPiece &rewrite, + const StringPiece &text, + string *out) const; + + // Escapes all potentially meaningful regexp characters in + // 'unquoted'. The returned string, used as a regular expression, + // will exactly match the original string. For example, + // 1.5-2.0? + // may become: + // 1\.5\-2\.0\? + // Note QuoteMeta behaves the same as perl's QuoteMeta function, + // *except* that it escapes the NUL character (\0) as backslash + 0, + // rather than backslash + NUL. + static string QuoteMeta(const StringPiece& unquoted); + + + /***** Generic matching interface *****/ + + // Type of match (TODO: Should be restructured as part of RE_Options) + enum Anchor { + UNANCHORED, // No anchoring + ANCHOR_START, // Anchor at start only + ANCHOR_BOTH // Anchor at start and end + }; + + // General matching routine. Stores the length of the match in + // "*consumed" if successful. + bool DoMatch(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const* args, int n) const; + + // Return the number of capturing subpatterns, or -1 if the + // regexp wasn't valid on construction. + int NumberOfCapturingGroups() const; + + // The default value for an argument, to indicate the end of the argument + // list. This must be used only in optional argument defaults. It should NOT + // be passed explicitly. Some people have tried to use it like this: + // + // FullMatch(x, y, &z, no_arg, &w); + // + // This is a mistake, and will not work. + static Arg no_arg; + + private: + + void Init(const string& pattern, const RE_Options* options); + void Cleanup(); + + // Match against "text", filling in "vec" (up to "vecsize" * 2/3) with + // pairs of integers for the beginning and end positions of matched + // text. The first pair corresponds to the entire matched text; + // subsequent pairs correspond, in order, to parentheses-captured + // matches. Returns the number of pairs (one more than the number of + // the last subpattern with a match) if matching was successful + // and zero if the match failed. + // I.e. for RE("(foo)|(bar)|(baz)") it will return 2, 3, and 4 when matching + // against "foo", "bar", and "baz" respectively. + // When matching RE("(foo)|hello") against "hello", it will return 1. + // But the values for all subpattern are filled in into "vec". + int TryMatch(const StringPiece& text, + int startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const; + + // Append the "rewrite" string, with backslash subsitutions from "text" + // and "vec", to string "out". + bool Rewrite(string *out, + const StringPiece& rewrite, + const StringPiece& text, + int *vec, + int veclen) const; + + // internal implementation for DoMatch + bool DoMatchImpl(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const args[], + int n, + int* vec, + int vecsize) const; + + // Compile the regexp for the specified anchoring mode + pcre* Compile(Anchor anchor); + + string pattern_; + RE_Options options_; + pcre* re_full_; // For full matches + pcre* re_partial_; // For partial matches + const string* error_; // Error indicator (or points to empty string) +}; + +} // namespace pcrecpp + +#endif /* _PCRECPP_H */ diff --git a/contrib/libs/pcre/pcrecpp/ya.make b/contrib/libs/pcre/pcrecpp/ya.make index 6b654d4420..c832b9e56e 100644 --- a/contrib/libs/pcre/pcrecpp/ya.make +++ b/contrib/libs/pcre/pcrecpp/ya.make @@ -1,7 +1,7 @@ # Generated by devtools/yamaker. -LIBRARY() - +LIBRARY() + WITHOUT_LICENSE_TEXTS() OWNER( @@ -11,12 +11,12 @@ OWNER( LICENSE(BSD-3-Clause) -PEERDIR( +PEERDIR( contrib/libs/pcre -) - +) + ADDINCL(contrib/libs/pcre) - + NO_COMPILER_WARNINGS() NO_UTIL() @@ -25,10 +25,10 @@ CFLAGS(-DHAVE_CONFIG_H) SRCDIR(contrib/libs/pcre) -SRCS( - pcre_scanner.cc - pcre_stringpiece.cc +SRCS( + pcre_scanner.cc + pcre_stringpiece.cc pcrecpp.cc -) - -END() +) + +END() diff --git a/contrib/libs/pcre/pcrecpp_internal.h b/contrib/libs/pcre/pcrecpp_internal.h index 552a1a6daa..827f9e04e2 100644 --- a/contrib/libs/pcre/pcrecpp_internal.h +++ b/contrib/libs/pcre/pcrecpp_internal.h @@ -1,71 +1,71 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* -Copyright (c) 2005, Google Inc. -All rights reserved. - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -#ifndef PCRECPP_INTERNAL_H -#define PCRECPP_INTERNAL_H - -/* When compiling a DLL for Windows, the exported symbols have to be declared -using some MS magic. I found some useful information on this web page: -http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the -information there, using __declspec(dllexport) without "extern" we have a -definition; with "extern" we have a declaration. The settings here override the -setting in pcre.h. We use: - - PCRECPP_EXP_DECL for declarations - PCRECPP_EXP_DEFN for definitions of exported functions - -*/ - -#ifndef PCRECPP_EXP_DECL -# ifdef _WIN32 -# ifndef PCRE_STATIC -# define PCRECPP_EXP_DECL extern __declspec(dllexport) -# define PCRECPP_EXP_DEFN __declspec(dllexport) -# else -# define PCRECPP_EXP_DECL extern -# define PCRECPP_EXP_DEFN -# endif -# else -# define PCRECPP_EXP_DECL extern -# define PCRECPP_EXP_DEFN -# endif -#endif - -#endif /* PCRECPP_INTERNAL_H */ - -/* End of pcrecpp_internal.h */ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* +Copyright (c) 2005, Google Inc. +All rights reserved. + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +#ifndef PCRECPP_INTERNAL_H +#define PCRECPP_INTERNAL_H + +/* When compiling a DLL for Windows, the exported symbols have to be declared +using some MS magic. I found some useful information on this web page: +http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the +information there, using __declspec(dllexport) without "extern" we have a +definition; with "extern" we have a declaration. The settings here override the +setting in pcre.h. We use: + + PCRECPP_EXP_DECL for declarations + PCRECPP_EXP_DEFN for definitions of exported functions + +*/ + +#ifndef PCRECPP_EXP_DECL +# ifdef _WIN32 +# ifndef PCRE_STATIC +# define PCRECPP_EXP_DECL extern __declspec(dllexport) +# define PCRECPP_EXP_DEFN __declspec(dllexport) +# else +# define PCRECPP_EXP_DECL extern +# define PCRECPP_EXP_DEFN +# endif +# else +# define PCRECPP_EXP_DECL extern +# define PCRECPP_EXP_DEFN +# endif +#endif + +#endif /* PCRECPP_INTERNAL_H */ + +/* End of pcrecpp_internal.h */ diff --git a/contrib/libs/pcre/pcrecpparg.h b/contrib/libs/pcre/pcrecpparg.h index a9915d09b5..9818347653 100644 --- a/contrib/libs/pcre/pcrecpparg.h +++ b/contrib/libs/pcre/pcrecpparg.h @@ -1,174 +1,174 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifndef _PCRECPPARG_H -#define _PCRECPPARG_H - -#include <stdlib.h> // for NULL -#include <string> - +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifndef _PCRECPPARG_H +#define _PCRECPPARG_H + +#include <stdlib.h> // for NULL +#include <string> + #include "pcre.h" - -namespace pcrecpp { - -class StringPiece; - -// Hex/Octal/Binary? - -// Special class for parsing into objects that define a ParseFrom() method -template <class T> -class _RE_MatchObject { - public: - static inline bool Parse(const char* str, int n, void* dest) { - if (dest == NULL) return true; - T* object = reinterpret_cast<T*>(dest); - return object->ParseFrom(str, n); - } -}; - -class PCRECPP_EXP_DEFN Arg { - public: - // Empty constructor so we can declare arrays of Arg - Arg(); - - // Constructor specially designed for NULL arguments - Arg(void*); - - typedef bool (*Parser)(const char* str, int n, void* dest); - -// Type-specific parsers -#define PCRE_MAKE_PARSER(type,name) \ - Arg(type* p) : arg_(p), parser_(name) { } \ - Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } - - - PCRE_MAKE_PARSER(char, parse_char); - PCRE_MAKE_PARSER(unsigned char, parse_uchar); - PCRE_MAKE_PARSER(short, parse_short); - PCRE_MAKE_PARSER(unsigned short, parse_ushort); - PCRE_MAKE_PARSER(int, parse_int); - PCRE_MAKE_PARSER(unsigned int, parse_uint); - PCRE_MAKE_PARSER(long, parse_long); - PCRE_MAKE_PARSER(unsigned long, parse_ulong); -#if 1 - PCRE_MAKE_PARSER(long long, parse_longlong); -#endif -#if 1 - PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); -#endif - PCRE_MAKE_PARSER(float, parse_float); - PCRE_MAKE_PARSER(double, parse_double); - PCRE_MAKE_PARSER(std::string, parse_string); - PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); - -#undef PCRE_MAKE_PARSER - - // Generic constructor - template <class T> Arg(T*, Parser parser); - // Generic constructor template - template <class T> Arg(T* p) - : arg_(p), parser_(_RE_MatchObject<T>::Parse) { - } - - // Parse the data - bool Parse(const char* str, int n) const; - - private: - void* arg_; - Parser parser_; - - static bool parse_null (const char* str, int n, void* dest); - static bool parse_char (const char* str, int n, void* dest); - static bool parse_uchar (const char* str, int n, void* dest); - static bool parse_float (const char* str, int n, void* dest); - static bool parse_double (const char* str, int n, void* dest); - static bool parse_string (const char* str, int n, void* dest); - static bool parse_stringpiece (const char* str, int n, void* dest); - -#define PCRE_DECLARE_INTEGER_PARSER(name) \ - private: \ - static bool parse_ ## name(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _radix( \ - const char* str, int n, void* dest, int radix); \ - public: \ - static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) - - PCRE_DECLARE_INTEGER_PARSER(short); - PCRE_DECLARE_INTEGER_PARSER(ushort); - PCRE_DECLARE_INTEGER_PARSER(int); - PCRE_DECLARE_INTEGER_PARSER(uint); - PCRE_DECLARE_INTEGER_PARSER(long); - PCRE_DECLARE_INTEGER_PARSER(ulong); - PCRE_DECLARE_INTEGER_PARSER(longlong); - PCRE_DECLARE_INTEGER_PARSER(ulonglong); - -#undef PCRE_DECLARE_INTEGER_PARSER -}; - -inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } -inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } - -inline bool Arg::Parse(const char* str, int n) const { - return (*parser_)(str, n, arg_); -} - -// This part of the parser, appropriate only for ints, deals with bases -#define MAKE_INTEGER_PARSER(type, name) \ - inline Arg Hex(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _hex); } \ - inline Arg Octal(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _octal); } \ - inline Arg CRadix(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _cradix); } - -MAKE_INTEGER_PARSER(short, short) /* */ -MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ -MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ -MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ -MAKE_INTEGER_PARSER(long, long) /* because they can cause */ -MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ -#if 1 /* the checking level is */ -MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ -#endif /* */ -#if 1 /* */ -MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ -#endif - -#undef PCRE_IS_SET -#undef PCRE_SET_OR_CLEAR -#undef MAKE_INTEGER_PARSER - -} // namespace pcrecpp - - -#endif /* _PCRECPPARG_H */ + +namespace pcrecpp { + +class StringPiece; + +// Hex/Octal/Binary? + +// Special class for parsing into objects that define a ParseFrom() method +template <class T> +class _RE_MatchObject { + public: + static inline bool Parse(const char* str, int n, void* dest) { + if (dest == NULL) return true; + T* object = reinterpret_cast<T*>(dest); + return object->ParseFrom(str, n); + } +}; + +class PCRECPP_EXP_DEFN Arg { + public: + // Empty constructor so we can declare arrays of Arg + Arg(); + + // Constructor specially designed for NULL arguments + Arg(void*); + + typedef bool (*Parser)(const char* str, int n, void* dest); + +// Type-specific parsers +#define PCRE_MAKE_PARSER(type,name) \ + Arg(type* p) : arg_(p), parser_(name) { } \ + Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } + + + PCRE_MAKE_PARSER(char, parse_char); + PCRE_MAKE_PARSER(unsigned char, parse_uchar); + PCRE_MAKE_PARSER(short, parse_short); + PCRE_MAKE_PARSER(unsigned short, parse_ushort); + PCRE_MAKE_PARSER(int, parse_int); + PCRE_MAKE_PARSER(unsigned int, parse_uint); + PCRE_MAKE_PARSER(long, parse_long); + PCRE_MAKE_PARSER(unsigned long, parse_ulong); +#if 1 + PCRE_MAKE_PARSER(long long, parse_longlong); +#endif +#if 1 + PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); +#endif + PCRE_MAKE_PARSER(float, parse_float); + PCRE_MAKE_PARSER(double, parse_double); + PCRE_MAKE_PARSER(std::string, parse_string); + PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); + +#undef PCRE_MAKE_PARSER + + // Generic constructor + template <class T> Arg(T*, Parser parser); + // Generic constructor template + template <class T> Arg(T* p) + : arg_(p), parser_(_RE_MatchObject<T>::Parse) { + } + + // Parse the data + bool Parse(const char* str, int n) const; + + private: + void* arg_; + Parser parser_; + + static bool parse_null (const char* str, int n, void* dest); + static bool parse_char (const char* str, int n, void* dest); + static bool parse_uchar (const char* str, int n, void* dest); + static bool parse_float (const char* str, int n, void* dest); + static bool parse_double (const char* str, int n, void* dest); + static bool parse_string (const char* str, int n, void* dest); + static bool parse_stringpiece (const char* str, int n, void* dest); + +#define PCRE_DECLARE_INTEGER_PARSER(name) \ + private: \ + static bool parse_ ## name(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _radix( \ + const char* str, int n, void* dest, int radix); \ + public: \ + static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) + + PCRE_DECLARE_INTEGER_PARSER(short); + PCRE_DECLARE_INTEGER_PARSER(ushort); + PCRE_DECLARE_INTEGER_PARSER(int); + PCRE_DECLARE_INTEGER_PARSER(uint); + PCRE_DECLARE_INTEGER_PARSER(long); + PCRE_DECLARE_INTEGER_PARSER(ulong); + PCRE_DECLARE_INTEGER_PARSER(longlong); + PCRE_DECLARE_INTEGER_PARSER(ulonglong); + +#undef PCRE_DECLARE_INTEGER_PARSER +}; + +inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } +inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } + +inline bool Arg::Parse(const char* str, int n) const { + return (*parser_)(str, n, arg_); +} + +// This part of the parser, appropriate only for ints, deals with bases +#define MAKE_INTEGER_PARSER(type, name) \ + inline Arg Hex(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _hex); } \ + inline Arg Octal(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _octal); } \ + inline Arg CRadix(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _cradix); } + +MAKE_INTEGER_PARSER(short, short) /* */ +MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ +MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ +MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ +MAKE_INTEGER_PARSER(long, long) /* because they can cause */ +MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ +#if 1 /* the checking level is */ +MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ +#endif /* */ +#if 1 /* */ +MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ +#endif + +#undef PCRE_IS_SET +#undef PCRE_SET_OR_CLEAR +#undef MAKE_INTEGER_PARSER + +} // namespace pcrecpp + + +#endif /* _PCRECPPARG_H */ diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index 68177a25bf..ed29bd14b9 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -233,7 +233,7 @@ namespace NActors { INTERCONNECT_PROXY_TCP = 12, INTERCONNECT_SESSION_TCP = 13, INTERCONNECT_COMMON = 171, - SELF_PING_ACTOR = 207, + SELF_PING_ACTOR = 207, TEST_ACTOR_RUNTIME = 283, INTERCONNECT_HANDSHAKE = 284, INTERCONNECT_POLLER = 285, @@ -248,11 +248,11 @@ namespace NActors { INTERCONNECT_PROXY_WRAPPER = 546, }; - using EActivityType = EActorActivity; - ui32 ActivityType; - + using EActivityType = EActorActivity; + ui32 ActivityType; + protected: - IActor(TReceiveFunc stateFunc, ui32 activityType = OTHER) + IActor(TReceiveFunc stateFunc, ui32 activityType = OTHER) : StateFunc(stateFunc) , SelfActorId(TActorId()) , ElapsedTicks(0) @@ -288,7 +288,7 @@ namespace NActors { } protected: - void SetActivityType(ui32 activityType) { + void SetActivityType(ui32 activityType) { ActivityType = activityType; } @@ -338,7 +338,7 @@ namespace NActors { void AddElapsedTicks(i64 ticks) { ElapsedTicks += ticks; } - auto GetActivityType() const { + auto GetActivityType() const { return ActivityType; } ui64 GetHandledEvents() const { @@ -393,28 +393,28 @@ namespace NActors { return TLocalProcessKeyState<TActorActivityTag>::GetInstance().GetNameByIndex(index); } - template <typename TDerived> - class TActor: public IActor { - private: - template <typename T, typename = const char*> - struct HasActorName: std::false_type { }; - template <typename T> - struct HasActorName<T, decltype((void)T::ActorName, (const char*)nullptr)>: std::true_type { }; - + template <typename TDerived> + class TActor: public IActor { + private: + template <typename T, typename = const char*> + struct HasActorName: std::false_type { }; + template <typename T> + struct HasActorName<T, decltype((void)T::ActorName, (const char*)nullptr)>: std::true_type { }; + static ui32 GetActivityTypeIndex() { - if constexpr(HasActorName<TDerived>::value) { - return TLocalProcessKey<TActorActivityTag, TDerived::ActorName>::GetIndex(); - } else { - using TActorActivity = decltype(((TDerived*)nullptr)->ActorActivityType()); - // if constexpr(std::is_enum<TActorActivity>::value) { - return TEnumProcessKey<TActorActivityTag, TActorActivity>::GetIndex( - TDerived::ActorActivityType()); - //} else { - // for int, ui32, ... - // return TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetIndex( - // static_cast<IActor::EActorActivity>(TDerived::ActorActivityType())); - //} - } + if constexpr(HasActorName<TDerived>::value) { + return TLocalProcessKey<TActorActivityTag, TDerived::ActorName>::GetIndex(); + } else { + using TActorActivity = decltype(((TDerived*)nullptr)->ActorActivityType()); + // if constexpr(std::is_enum<TActorActivity>::value) { + return TEnumProcessKey<TActorActivityTag, TActorActivity>::GetIndex( + TDerived::ActorActivityType()); + //} else { + // for int, ui32, ... + // return TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetIndex( + // static_cast<IActor::EActorActivity>(TDerived::ActorActivityType())); + //} + } } protected: @@ -423,17 +423,17 @@ namespace NActors { return EActorActivity::OTHER; } //*/ - // static constexpr char ActorName[] = "UNNAMED"; - + // static constexpr char ActorName[] = "UNNAMED"; + TActor(void (TDerived::*func)(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx), ui32 activityType = GetActivityTypeIndex()) : IActor(static_cast<TReceiveFunc>(func), activityType) - { } + { } public: typedef TDerived TThis; }; - + #define STFUNC_SIG TAutoPtr< ::NActors::IEventHandle>&ev, const ::NActors::TActorContext &ctx #define STATEFN_SIG TAutoPtr<::NActors::IEventHandle>& ev #define STFUNC(funcName) void funcName(TAutoPtr< ::NActors::IEventHandle>& ev, const ::NActors::TActorContext& ctx) diff --git a/library/cpp/actors/core/actor_coroutine.h b/library/cpp/actors/core/actor_coroutine.h index 8f1450c434..6bcb768eaf 100644 --- a/library/cpp/actors/core/actor_coroutine.h +++ b/library/cpp/actors/core/actor_coroutine.h @@ -154,7 +154,7 @@ namespace NActors { THolder<TActorCoroImpl> Impl; public: - TActorCoro(THolder<TActorCoroImpl> impl, ui32 activityType = IActor::ACTORLIB_COMMON) + TActorCoro(THolder<TActorCoroImpl> impl, ui32 activityType = IActor::ACTORLIB_COMMON) : IActor(static_cast<TReceiveFunc>(&TActorCoro::StateFunc), activityType) , Impl(std::move(impl)) {} diff --git a/library/cpp/actors/core/actor_ut.cpp b/library/cpp/actors/core/actor_ut.cpp index a5d43b4619..e1b765ec72 100644 --- a/library/cpp/actors/core/actor_ut.cpp +++ b/library/cpp/actors/core/actor_ut.cpp @@ -55,7 +55,7 @@ Y_UNIT_TEST_SUITE(ActorBenchmark) { class TSendReceiveActor : public TActorBootstrapped<TSendReceiveActor> { public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return ACTORLIB_COMMON; } @@ -525,12 +525,12 @@ Y_UNIT_TEST_SUITE(TestDecorator) { } }; - struct TTestActor : TActorBootstrapped<TTestActor> { - static constexpr char ActorName[] = "TestActor"; + struct TTestActor : TActorBootstrapped<TTestActor> { + static constexpr char ActorName[] = "TestActor"; void Bootstrap() { - const auto& activityTypeIndex = GetActivityType(); + const auto& activityTypeIndex = GetActivityType(); Y_ENSURE(activityTypeIndex < GetActivityTypeCount()); Y_ENSURE(GetActivityTypeName(activityTypeIndex) == "TestActor"); PassAway(); @@ -566,13 +566,13 @@ Y_UNIT_TEST_SUITE(TestDecorator) { actorSystem.Stop(); UNIT_ASSERT(pongCounter == 2 && pingCounter == 2); } - - Y_UNIT_TEST(LocalProcessKey) { - static constexpr char ActorName[] = "TestActor"; - - UNIT_ASSERT((TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetName(IActor::INTERCONNECT_PROXY_TCP) == "INTERCONNECT_PROXY_TCP")); - - UNIT_ASSERT((TLocalProcessKey<TActorActivityTag, ActorName>::GetName() == ActorName)); - UNIT_ASSERT((TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetIndex(IActor::INTERCONNECT_PROXY_TCP) == IActor::INTERCONNECT_PROXY_TCP)); - } + + Y_UNIT_TEST(LocalProcessKey) { + static constexpr char ActorName[] = "TestActor"; + + UNIT_ASSERT((TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetName(IActor::INTERCONNECT_PROXY_TCP) == "INTERCONNECT_PROXY_TCP")); + + UNIT_ASSERT((TLocalProcessKey<TActorActivityTag, ActorName>::GetName() == ActorName)); + UNIT_ASSERT((TEnumProcessKey<TActorActivityTag, IActor::EActorActivity>::GetIndex(IActor::INTERCONNECT_PROXY_TCP) == IActor::INTERCONNECT_PROXY_TCP)); + } } diff --git a/library/cpp/actors/core/invoke.h b/library/cpp/actors/core/invoke.h index 106097abc9..931a9767dd 100644 --- a/library/cpp/actors/core/invoke.h +++ b/library/cpp/actors/core/invoke.h @@ -77,14 +77,14 @@ namespace NActors { // actor's class. But `complete` handler is invoked in parent context and can use its contents. Do not forget to // handle TEvInvokeResult event by calling Process/GetResult method, whichever is necessary. - template<typename TCallback, typename TCompletion, ui32 Activity> + template<typename TCallback, typename TCompletion, ui32 Activity> class TInvokeActor : public TActorBootstrapped<TInvokeActor<TCallback, TCompletion, Activity>> { TCallback Callback; TCompletion Complete; public: - static constexpr auto ActorActivityType() { - return static_cast<IActor::EActorActivity>(Activity); + static constexpr auto ActorActivityType() { + return static_cast<IActor::EActorActivity>(Activity); } TInvokeActor(TCallback&& callback, TCompletion&& complete) @@ -101,7 +101,7 @@ namespace NActors { } }; - template<ui32 Activity, typename TCallback, typename TCompletion> + template<ui32 Activity, typename TCallback, typename TCompletion> std::unique_ptr<IActor> CreateInvokeActor(TCallback&& callback, TCompletion&& complete) { return std::make_unique<TInvokeActor<std::decay_t<TCallback>, std::decay_t<TCompletion>, Activity>>( std::forward<TCallback>(callback), std::forward<TCompletion>(complete)); diff --git a/library/cpp/actors/core/ya.make b/library/cpp/actors/core/ya.make index 9a5ab29e51..880a9d00db 100644 --- a/library/cpp/actors/core/ya.make +++ b/library/cpp/actors/core/ya.make @@ -100,7 +100,7 @@ SRCS( ) GENERATE_ENUM_SERIALIZATION(defs.h) -GENERATE_ENUM_SERIALIZATION(actor.h) +GENERATE_ENUM_SERIALIZATION(actor.h) PEERDIR( library/cpp/actors/memory_log diff --git a/library/cpp/actors/helpers/pool_stats_collector.h b/library/cpp/actors/helpers/pool_stats_collector.h index 78c842900e..61d0b45780 100644 --- a/library/cpp/actors/helpers/pool_stats_collector.h +++ b/library/cpp/actors/helpers/pool_stats_collector.h @@ -1,314 +1,314 @@ -#pragma once - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/actors/core/executor_thread.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> - -#include <util/generic/vector.h> -#include <util/generic/xrange.h> -#include <util/string/printf.h> - -namespace NActors { - -// Periodically collects stats from executor threads and exposes them as mon counters -class TStatsCollectingActor : public TActorBootstrapped<TStatsCollectingActor> { -private: - struct THistogramCounters { - void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, ui64 maxVal) { - for (size_t i = 0; (1ull<<i) <= maxVal; ++i) { - TString bucketName = ToString(1ull<<i) + " " + unit; - Buckets.push_back(group->GetSubgroup("sensor", baseName)->GetNamedCounter("range", bucketName, true)); - } - Buckets.push_back(group->GetSubgroup("sensor", baseName)->GetNamedCounter("range", "INF", true)); - } - - void Set(const TLogHistogram& data) { - ui32 i = 0; - for (;i < Y_ARRAY_SIZE(data.Buckets) && i < Buckets.size()-1; ++i) - *Buckets[i] = data.Buckets[i]; - ui64 last = 0; - for (;i < Y_ARRAY_SIZE(data.Buckets); ++i) - last += data.Buckets[i]; - *Buckets.back() = last; - } - - void Set(const TLogHistogram& data, double factor) { - ui32 i = 0; - for (;i < Y_ARRAY_SIZE(data.Buckets) && i < Buckets.size()-1; ++i) - *Buckets[i] = data.Buckets[i]*factor; - ui64 last = 0; - for (;i < Y_ARRAY_SIZE(data.Buckets); ++i) - last += data.Buckets[i]; - *Buckets.back() = last*factor; - } - - private: - TVector<NMonitoring::TDynamicCounters::TCounterPtr> Buckets; - }; - - struct TActivityStats { - void Init(NMonitoring::TDynamicCounterPtr group) { - Group = group; - - ElapsedMicrosecByActivityBuckets.resize(GetActivityTypeCount()); - ReceivedEventsByActivityBuckets.resize(GetActivityTypeCount()); - ActorsAliveByActivityBuckets.resize(GetActivityTypeCount()); - ScheduledEventsByActivityBuckets.resize(GetActivityTypeCount()); - } - - void Set(const TExecutorThreadStats& stats) { - for (ui32 i : xrange(stats.MaxActivityType())) { - Y_VERIFY(i < GetActivityTypeCount()); - ui64 ticks = stats.ElapsedTicksByActivity[i]; - ui64 events = stats.ReceivedEventsByActivity[i]; - ui64 actors = stats.ActorsAliveByActivity[i]; - ui64 scheduled = stats.ScheduledEventsByActivity[i]; - - if (!ActorsAliveByActivityBuckets[i]) { - if (ticks || events || actors || scheduled) { - InitCountersForActivity(i); - } else { - continue; - } - } - - *ElapsedMicrosecByActivityBuckets[i] = ::NHPTimer::GetSeconds(ticks)*1000000; - *ReceivedEventsByActivityBuckets[i] = events; - *ActorsAliveByActivityBuckets[i] = actors; - *ScheduledEventsByActivityBuckets[i] = scheduled; - } - } - - private: - void InitCountersForActivity(ui32 activityType) { - Y_VERIFY(activityType < GetActivityTypeCount()); - - auto bucketName = TString(GetActivityTypeName(activityType)); - - ElapsedMicrosecByActivityBuckets[activityType] = - Group->GetSubgroup("sensor", "ElapsedMicrosecByActivity")->GetNamedCounter("activity", bucketName, true); - ReceivedEventsByActivityBuckets[activityType] = - Group->GetSubgroup("sensor", "ReceivedEventsByActivity")->GetNamedCounter("activity", bucketName, true); - ActorsAliveByActivityBuckets[activityType] = - Group->GetSubgroup("sensor", "ActorsAliveByActivity")->GetNamedCounter("activity", bucketName, false); - ScheduledEventsByActivityBuckets[activityType] = - Group->GetSubgroup("sensor", "ScheduledEventsByActivity")->GetNamedCounter("activity", bucketName, true); - } - - private: - NMonitoring::TDynamicCounterPtr Group; - - TVector<NMonitoring::TDynamicCounters::TCounterPtr> ElapsedMicrosecByActivityBuckets; - TVector<NMonitoring::TDynamicCounters::TCounterPtr> ReceivedEventsByActivityBuckets; - TVector<NMonitoring::TDynamicCounters::TCounterPtr> ActorsAliveByActivityBuckets; - TVector<NMonitoring::TDynamicCounters::TCounterPtr> ScheduledEventsByActivityBuckets; - }; - - struct TExecutorPoolCounters { - TIntrusivePtr<NMonitoring::TDynamicCounters> PoolGroup; - - NMonitoring::TDynamicCounters::TCounterPtr SentEvents; - NMonitoring::TDynamicCounters::TCounterPtr ReceivedEvents; - NMonitoring::TDynamicCounters::TCounterPtr PreemptedEvents; - NMonitoring::TDynamicCounters::TCounterPtr NonDeliveredEvents; - NMonitoring::TDynamicCounters::TCounterPtr DestroyedActors; - NMonitoring::TDynamicCounters::TCounterPtr EmptyMailboxActivation; - NMonitoring::TDynamicCounters::TCounterPtr CpuMicrosec; - NMonitoring::TDynamicCounters::TCounterPtr ElapsedMicrosec; - NMonitoring::TDynamicCounters::TCounterPtr ParkedMicrosec; - NMonitoring::TDynamicCounters::TCounterPtr ActorRegistrations; - NMonitoring::TDynamicCounters::TCounterPtr ActorsAlive; - NMonitoring::TDynamicCounters::TCounterPtr AllocatedMailboxes; - NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutBySoftPreemption; - NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutByTime; - NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutByEventCount; - - THistogramCounters LegacyActivationTimeHistogram; - NMonitoring::THistogramPtr ActivationTimeHistogram; - THistogramCounters LegacyEventDeliveryTimeHistogram; - NMonitoring::THistogramPtr EventDeliveryTimeHistogram; - THistogramCounters LegacyEventProcessingCountHistogram; - NMonitoring::THistogramPtr EventProcessingCountHistogram; - THistogramCounters LegacyEventProcessingTimeHistogram; - NMonitoring::THistogramPtr EventProcessingTimeHistogram; - - TActivityStats ActivityStats; - NMonitoring::TDynamicCounters::TCounterPtr MaxUtilizationTime; - - double Usage = 0; - double LastElapsedSeconds = 0; - THPTimer UsageTimer; - TString Name; - ui32 Threads; - - void Init(NMonitoring::TDynamicCounters* group, const TString& poolName, ui32 threads) { - LastElapsedSeconds = 0; - Usage = 0; - UsageTimer.Reset(); - Name = poolName; - Threads = threads; - - PoolGroup = group->GetSubgroup("execpool", poolName); - - SentEvents = PoolGroup->GetCounter("SentEvents", true); - ReceivedEvents = PoolGroup->GetCounter("ReceivedEvents", true); - PreemptedEvents = PoolGroup->GetCounter("PreemptedEvents", true); - NonDeliveredEvents = PoolGroup->GetCounter("NonDeliveredEvents", true); - DestroyedActors = PoolGroup->GetCounter("DestroyedActors", true); - CpuMicrosec = PoolGroup->GetCounter("CpuMicrosec", true); - ElapsedMicrosec = PoolGroup->GetCounter("ElapsedMicrosec", true); - ParkedMicrosec = PoolGroup->GetCounter("ParkedMicrosec", true); - EmptyMailboxActivation = PoolGroup->GetCounter("EmptyMailboxActivation", true); - ActorRegistrations = PoolGroup->GetCounter("ActorRegistrations", true); - ActorsAlive = PoolGroup->GetCounter("ActorsAlive", false); - AllocatedMailboxes = PoolGroup->GetCounter("AllocatedMailboxes", false); - MailboxPushedOutBySoftPreemption = PoolGroup->GetCounter("MailboxPushedOutBySoftPreemption", true); - MailboxPushedOutByTime = PoolGroup->GetCounter("MailboxPushedOutByTime", true); - MailboxPushedOutByEventCount = PoolGroup->GetCounter("MailboxPushedOutByEventCount", true); - - LegacyActivationTimeHistogram.Init(PoolGroup.Get(), "ActivationTime", "usec", 5*1000*1000); - ActivationTimeHistogram = PoolGroup->GetHistogram( - "ActivationTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); - LegacyEventDeliveryTimeHistogram.Init(PoolGroup.Get(), "EventDeliveryTime", "usec", 5*1000*1000); - EventDeliveryTimeHistogram = PoolGroup->GetHistogram( - "EventDeliveryTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); - LegacyEventProcessingCountHistogram.Init(PoolGroup.Get(), "EventProcessingCount", "usec", 5*1000*1000); - EventProcessingCountHistogram = PoolGroup->GetHistogram( - "EventProcessingCountUs", NMonitoring::ExponentialHistogram(24, 2, 1)); - LegacyEventProcessingTimeHistogram.Init(PoolGroup.Get(), "EventProcessingTime", "usec", 5*1000*1000); - EventProcessingTimeHistogram = PoolGroup->GetHistogram( - "EventProcessingTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); - - ActivityStats.Init(PoolGroup.Get()); - - MaxUtilizationTime = PoolGroup->GetCounter("MaxUtilizationTime", true); - } - - void Set(const TExecutorPoolStats& poolStats, const TExecutorThreadStats& stats, ui32 numThreads) { -#ifdef ACTORSLIB_COLLECT_EXEC_STATS - *SentEvents = stats.SentEvents; - *ReceivedEvents = stats.ReceivedEvents; - *PreemptedEvents = stats.PreemptedEvents; - *NonDeliveredEvents = stats.NonDeliveredEvents; - *DestroyedActors = stats.PoolDestroyedActors; - *EmptyMailboxActivation = stats.EmptyMailboxActivation; - *CpuMicrosec = stats.CpuNs / 1000; - *ElapsedMicrosec = ::NHPTimer::GetSeconds(stats.ElapsedTicks)*1000000; - *ParkedMicrosec = ::NHPTimer::GetSeconds(stats.ParkedTicks)*1000000; - *ActorRegistrations = stats.PoolActorRegistrations; - *ActorsAlive = stats.PoolActorRegistrations - stats.PoolDestroyedActors; - *AllocatedMailboxes = stats.PoolAllocatedMailboxes; - *MailboxPushedOutBySoftPreemption = stats.MailboxPushedOutBySoftPreemption; - *MailboxPushedOutByTime = stats.MailboxPushedOutByTime; - *MailboxPushedOutByEventCount = stats.MailboxPushedOutByEventCount; - - LegacyActivationTimeHistogram.Set(stats.ActivationTimeHistogram); - ActivationTimeHistogram->Reset(); - ActivationTimeHistogram->Collect(stats.ActivationTimeHistogram); - - LegacyEventDeliveryTimeHistogram.Set(stats.EventDeliveryTimeHistogram); - EventDeliveryTimeHistogram->Reset(); - EventDeliveryTimeHistogram->Collect(stats.EventDeliveryTimeHistogram); - - LegacyEventProcessingCountHistogram.Set(stats.EventProcessingCountHistogram); - EventProcessingCountHistogram->Reset(); - EventProcessingCountHistogram->Collect(stats.EventProcessingCountHistogram); - - double toMicrosec = 1000000 / NHPTimer::GetClockRate(); - LegacyEventProcessingTimeHistogram.Set(stats.EventProcessingTimeHistogram, toMicrosec); - EventProcessingTimeHistogram->Reset(); - for (ui32 i = 0; i < stats.EventProcessingTimeHistogram.Count(); ++i) { - EventProcessingTimeHistogram->Collect( - stats.EventProcessingTimeHistogram.UpperBound(i), - stats.EventProcessingTimeHistogram.Value(i) * toMicrosec); - } - - ActivityStats.Set(stats); - - *MaxUtilizationTime = poolStats.MaxUtilizationTime; - - double seconds = UsageTimer.PassedReset(); - - // TODO[serxa]: It doesn't account for contention. Use 1 - parkedTicksDelta / seconds / numThreads KIKIMR-11916 - const double elapsed = NHPTimer::GetSeconds(stats.ElapsedTicks); - const double currentUsage = numThreads > 0 ? ((elapsed - LastElapsedSeconds) / seconds / numThreads) : 0; - LastElapsedSeconds = elapsed; - - // update usage factor according to smoothness - const double smoothness = 0.5; - Usage = currentUsage * smoothness + Usage * (1.0 - smoothness); -#else - Y_UNUSED(poolStats); - Y_UNUSED(stats); - Y_UNUSED(numThreads); -#endif - } - }; - -public: - static constexpr IActor::EActivityType ActorActivityType() { - return IActor::ACTORLIB_STATS; - } - - TStatsCollectingActor( - ui32 intervalSec, - const TActorSystemSetup& setup, - NMonitoring::TDynamicCounterPtr counters) - : IntervalSec(intervalSec) - , Counters(counters) - { - PoolCounters.resize(setup.GetExecutorsCount()); - for (size_t poolId = 0; poolId < PoolCounters.size(); ++poolId) { - PoolCounters[poolId].Init(Counters.Get(), setup.GetPoolName(poolId), setup.GetThreads(poolId)); - } - } - - void Bootstrap(const TActorContext& ctx) { - ctx.Schedule(TDuration::Seconds(IntervalSec), new TEvents::TEvWakeup()); - Become(&TThis::StateWork); - } - - STFUNC(StateWork) { - switch (ev->GetTypeRewrite()) { - CFunc(TEvents::TSystem::Wakeup, Wakeup); - } - } - -private: - virtual void OnWakeup(const TActorContext &ctx) { - Y_UNUSED(ctx); - } - - void Wakeup(const TActorContext &ctx) { - for (size_t poolId = 0; poolId < PoolCounters.size(); ++poolId) { - TVector<TExecutorThreadStats> stats; - TExecutorPoolStats poolStats; - ctx.ExecutorThread.ActorSystem->GetPoolStats(poolId, poolStats, stats); - SetAggregatedCounters(PoolCounters[poolId], poolStats, stats); - } - - OnWakeup(ctx); - - ctx.Schedule(TDuration::Seconds(IntervalSec), new TEvents::TEvWakeup()); - } - - void SetAggregatedCounters(TExecutorPoolCounters& poolCounters, TExecutorPoolStats& poolStats, TVector<TExecutorThreadStats>& stats) { - // Sum all per-thread counters into the 0th element - for (ui32 idx = 1; idx < stats.size(); ++idx) { - stats[0].Aggregate(stats[idx]); - } - if (stats.size()) { - poolCounters.Set(poolStats, stats[0], stats.size() - 1); - } - } - -protected: - const ui32 IntervalSec; - NMonitoring::TDynamicCounterPtr Counters; - - TVector<TExecutorPoolCounters> PoolCounters; -}; - -} // NActors +#pragma once + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/executor_thread.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> + +#include <util/generic/vector.h> +#include <util/generic/xrange.h> +#include <util/string/printf.h> + +namespace NActors { + +// Periodically collects stats from executor threads and exposes them as mon counters +class TStatsCollectingActor : public TActorBootstrapped<TStatsCollectingActor> { +private: + struct THistogramCounters { + void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, ui64 maxVal) { + for (size_t i = 0; (1ull<<i) <= maxVal; ++i) { + TString bucketName = ToString(1ull<<i) + " " + unit; + Buckets.push_back(group->GetSubgroup("sensor", baseName)->GetNamedCounter("range", bucketName, true)); + } + Buckets.push_back(group->GetSubgroup("sensor", baseName)->GetNamedCounter("range", "INF", true)); + } + + void Set(const TLogHistogram& data) { + ui32 i = 0; + for (;i < Y_ARRAY_SIZE(data.Buckets) && i < Buckets.size()-1; ++i) + *Buckets[i] = data.Buckets[i]; + ui64 last = 0; + for (;i < Y_ARRAY_SIZE(data.Buckets); ++i) + last += data.Buckets[i]; + *Buckets.back() = last; + } + + void Set(const TLogHistogram& data, double factor) { + ui32 i = 0; + for (;i < Y_ARRAY_SIZE(data.Buckets) && i < Buckets.size()-1; ++i) + *Buckets[i] = data.Buckets[i]*factor; + ui64 last = 0; + for (;i < Y_ARRAY_SIZE(data.Buckets); ++i) + last += data.Buckets[i]; + *Buckets.back() = last*factor; + } + + private: + TVector<NMonitoring::TDynamicCounters::TCounterPtr> Buckets; + }; + + struct TActivityStats { + void Init(NMonitoring::TDynamicCounterPtr group) { + Group = group; + + ElapsedMicrosecByActivityBuckets.resize(GetActivityTypeCount()); + ReceivedEventsByActivityBuckets.resize(GetActivityTypeCount()); + ActorsAliveByActivityBuckets.resize(GetActivityTypeCount()); + ScheduledEventsByActivityBuckets.resize(GetActivityTypeCount()); + } + + void Set(const TExecutorThreadStats& stats) { + for (ui32 i : xrange(stats.MaxActivityType())) { + Y_VERIFY(i < GetActivityTypeCount()); + ui64 ticks = stats.ElapsedTicksByActivity[i]; + ui64 events = stats.ReceivedEventsByActivity[i]; + ui64 actors = stats.ActorsAliveByActivity[i]; + ui64 scheduled = stats.ScheduledEventsByActivity[i]; + + if (!ActorsAliveByActivityBuckets[i]) { + if (ticks || events || actors || scheduled) { + InitCountersForActivity(i); + } else { + continue; + } + } + + *ElapsedMicrosecByActivityBuckets[i] = ::NHPTimer::GetSeconds(ticks)*1000000; + *ReceivedEventsByActivityBuckets[i] = events; + *ActorsAliveByActivityBuckets[i] = actors; + *ScheduledEventsByActivityBuckets[i] = scheduled; + } + } + + private: + void InitCountersForActivity(ui32 activityType) { + Y_VERIFY(activityType < GetActivityTypeCount()); + + auto bucketName = TString(GetActivityTypeName(activityType)); + + ElapsedMicrosecByActivityBuckets[activityType] = + Group->GetSubgroup("sensor", "ElapsedMicrosecByActivity")->GetNamedCounter("activity", bucketName, true); + ReceivedEventsByActivityBuckets[activityType] = + Group->GetSubgroup("sensor", "ReceivedEventsByActivity")->GetNamedCounter("activity", bucketName, true); + ActorsAliveByActivityBuckets[activityType] = + Group->GetSubgroup("sensor", "ActorsAliveByActivity")->GetNamedCounter("activity", bucketName, false); + ScheduledEventsByActivityBuckets[activityType] = + Group->GetSubgroup("sensor", "ScheduledEventsByActivity")->GetNamedCounter("activity", bucketName, true); + } + + private: + NMonitoring::TDynamicCounterPtr Group; + + TVector<NMonitoring::TDynamicCounters::TCounterPtr> ElapsedMicrosecByActivityBuckets; + TVector<NMonitoring::TDynamicCounters::TCounterPtr> ReceivedEventsByActivityBuckets; + TVector<NMonitoring::TDynamicCounters::TCounterPtr> ActorsAliveByActivityBuckets; + TVector<NMonitoring::TDynamicCounters::TCounterPtr> ScheduledEventsByActivityBuckets; + }; + + struct TExecutorPoolCounters { + TIntrusivePtr<NMonitoring::TDynamicCounters> PoolGroup; + + NMonitoring::TDynamicCounters::TCounterPtr SentEvents; + NMonitoring::TDynamicCounters::TCounterPtr ReceivedEvents; + NMonitoring::TDynamicCounters::TCounterPtr PreemptedEvents; + NMonitoring::TDynamicCounters::TCounterPtr NonDeliveredEvents; + NMonitoring::TDynamicCounters::TCounterPtr DestroyedActors; + NMonitoring::TDynamicCounters::TCounterPtr EmptyMailboxActivation; + NMonitoring::TDynamicCounters::TCounterPtr CpuMicrosec; + NMonitoring::TDynamicCounters::TCounterPtr ElapsedMicrosec; + NMonitoring::TDynamicCounters::TCounterPtr ParkedMicrosec; + NMonitoring::TDynamicCounters::TCounterPtr ActorRegistrations; + NMonitoring::TDynamicCounters::TCounterPtr ActorsAlive; + NMonitoring::TDynamicCounters::TCounterPtr AllocatedMailboxes; + NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutBySoftPreemption; + NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutByTime; + NMonitoring::TDynamicCounters::TCounterPtr MailboxPushedOutByEventCount; + + THistogramCounters LegacyActivationTimeHistogram; + NMonitoring::THistogramPtr ActivationTimeHistogram; + THistogramCounters LegacyEventDeliveryTimeHistogram; + NMonitoring::THistogramPtr EventDeliveryTimeHistogram; + THistogramCounters LegacyEventProcessingCountHistogram; + NMonitoring::THistogramPtr EventProcessingCountHistogram; + THistogramCounters LegacyEventProcessingTimeHistogram; + NMonitoring::THistogramPtr EventProcessingTimeHistogram; + + TActivityStats ActivityStats; + NMonitoring::TDynamicCounters::TCounterPtr MaxUtilizationTime; + + double Usage = 0; + double LastElapsedSeconds = 0; + THPTimer UsageTimer; + TString Name; + ui32 Threads; + + void Init(NMonitoring::TDynamicCounters* group, const TString& poolName, ui32 threads) { + LastElapsedSeconds = 0; + Usage = 0; + UsageTimer.Reset(); + Name = poolName; + Threads = threads; + + PoolGroup = group->GetSubgroup("execpool", poolName); + + SentEvents = PoolGroup->GetCounter("SentEvents", true); + ReceivedEvents = PoolGroup->GetCounter("ReceivedEvents", true); + PreemptedEvents = PoolGroup->GetCounter("PreemptedEvents", true); + NonDeliveredEvents = PoolGroup->GetCounter("NonDeliveredEvents", true); + DestroyedActors = PoolGroup->GetCounter("DestroyedActors", true); + CpuMicrosec = PoolGroup->GetCounter("CpuMicrosec", true); + ElapsedMicrosec = PoolGroup->GetCounter("ElapsedMicrosec", true); + ParkedMicrosec = PoolGroup->GetCounter("ParkedMicrosec", true); + EmptyMailboxActivation = PoolGroup->GetCounter("EmptyMailboxActivation", true); + ActorRegistrations = PoolGroup->GetCounter("ActorRegistrations", true); + ActorsAlive = PoolGroup->GetCounter("ActorsAlive", false); + AllocatedMailboxes = PoolGroup->GetCounter("AllocatedMailboxes", false); + MailboxPushedOutBySoftPreemption = PoolGroup->GetCounter("MailboxPushedOutBySoftPreemption", true); + MailboxPushedOutByTime = PoolGroup->GetCounter("MailboxPushedOutByTime", true); + MailboxPushedOutByEventCount = PoolGroup->GetCounter("MailboxPushedOutByEventCount", true); + + LegacyActivationTimeHistogram.Init(PoolGroup.Get(), "ActivationTime", "usec", 5*1000*1000); + ActivationTimeHistogram = PoolGroup->GetHistogram( + "ActivationTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); + LegacyEventDeliveryTimeHistogram.Init(PoolGroup.Get(), "EventDeliveryTime", "usec", 5*1000*1000); + EventDeliveryTimeHistogram = PoolGroup->GetHistogram( + "EventDeliveryTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); + LegacyEventProcessingCountHistogram.Init(PoolGroup.Get(), "EventProcessingCount", "usec", 5*1000*1000); + EventProcessingCountHistogram = PoolGroup->GetHistogram( + "EventProcessingCountUs", NMonitoring::ExponentialHistogram(24, 2, 1)); + LegacyEventProcessingTimeHistogram.Init(PoolGroup.Get(), "EventProcessingTime", "usec", 5*1000*1000); + EventProcessingTimeHistogram = PoolGroup->GetHistogram( + "EventProcessingTimeUs", NMonitoring::ExponentialHistogram(24, 2, 1)); + + ActivityStats.Init(PoolGroup.Get()); + + MaxUtilizationTime = PoolGroup->GetCounter("MaxUtilizationTime", true); + } + + void Set(const TExecutorPoolStats& poolStats, const TExecutorThreadStats& stats, ui32 numThreads) { +#ifdef ACTORSLIB_COLLECT_EXEC_STATS + *SentEvents = stats.SentEvents; + *ReceivedEvents = stats.ReceivedEvents; + *PreemptedEvents = stats.PreemptedEvents; + *NonDeliveredEvents = stats.NonDeliveredEvents; + *DestroyedActors = stats.PoolDestroyedActors; + *EmptyMailboxActivation = stats.EmptyMailboxActivation; + *CpuMicrosec = stats.CpuNs / 1000; + *ElapsedMicrosec = ::NHPTimer::GetSeconds(stats.ElapsedTicks)*1000000; + *ParkedMicrosec = ::NHPTimer::GetSeconds(stats.ParkedTicks)*1000000; + *ActorRegistrations = stats.PoolActorRegistrations; + *ActorsAlive = stats.PoolActorRegistrations - stats.PoolDestroyedActors; + *AllocatedMailboxes = stats.PoolAllocatedMailboxes; + *MailboxPushedOutBySoftPreemption = stats.MailboxPushedOutBySoftPreemption; + *MailboxPushedOutByTime = stats.MailboxPushedOutByTime; + *MailboxPushedOutByEventCount = stats.MailboxPushedOutByEventCount; + + LegacyActivationTimeHistogram.Set(stats.ActivationTimeHistogram); + ActivationTimeHistogram->Reset(); + ActivationTimeHistogram->Collect(stats.ActivationTimeHistogram); + + LegacyEventDeliveryTimeHistogram.Set(stats.EventDeliveryTimeHistogram); + EventDeliveryTimeHistogram->Reset(); + EventDeliveryTimeHistogram->Collect(stats.EventDeliveryTimeHistogram); + + LegacyEventProcessingCountHistogram.Set(stats.EventProcessingCountHistogram); + EventProcessingCountHistogram->Reset(); + EventProcessingCountHistogram->Collect(stats.EventProcessingCountHistogram); + + double toMicrosec = 1000000 / NHPTimer::GetClockRate(); + LegacyEventProcessingTimeHistogram.Set(stats.EventProcessingTimeHistogram, toMicrosec); + EventProcessingTimeHistogram->Reset(); + for (ui32 i = 0; i < stats.EventProcessingTimeHistogram.Count(); ++i) { + EventProcessingTimeHistogram->Collect( + stats.EventProcessingTimeHistogram.UpperBound(i), + stats.EventProcessingTimeHistogram.Value(i) * toMicrosec); + } + + ActivityStats.Set(stats); + + *MaxUtilizationTime = poolStats.MaxUtilizationTime; + + double seconds = UsageTimer.PassedReset(); + + // TODO[serxa]: It doesn't account for contention. Use 1 - parkedTicksDelta / seconds / numThreads KIKIMR-11916 + const double elapsed = NHPTimer::GetSeconds(stats.ElapsedTicks); + const double currentUsage = numThreads > 0 ? ((elapsed - LastElapsedSeconds) / seconds / numThreads) : 0; + LastElapsedSeconds = elapsed; + + // update usage factor according to smoothness + const double smoothness = 0.5; + Usage = currentUsage * smoothness + Usage * (1.0 - smoothness); +#else + Y_UNUSED(poolStats); + Y_UNUSED(stats); + Y_UNUSED(numThreads); +#endif + } + }; + +public: + static constexpr IActor::EActivityType ActorActivityType() { + return IActor::ACTORLIB_STATS; + } + + TStatsCollectingActor( + ui32 intervalSec, + const TActorSystemSetup& setup, + NMonitoring::TDynamicCounterPtr counters) + : IntervalSec(intervalSec) + , Counters(counters) + { + PoolCounters.resize(setup.GetExecutorsCount()); + for (size_t poolId = 0; poolId < PoolCounters.size(); ++poolId) { + PoolCounters[poolId].Init(Counters.Get(), setup.GetPoolName(poolId), setup.GetThreads(poolId)); + } + } + + void Bootstrap(const TActorContext& ctx) { + ctx.Schedule(TDuration::Seconds(IntervalSec), new TEvents::TEvWakeup()); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + CFunc(TEvents::TSystem::Wakeup, Wakeup); + } + } + +private: + virtual void OnWakeup(const TActorContext &ctx) { + Y_UNUSED(ctx); + } + + void Wakeup(const TActorContext &ctx) { + for (size_t poolId = 0; poolId < PoolCounters.size(); ++poolId) { + TVector<TExecutorThreadStats> stats; + TExecutorPoolStats poolStats; + ctx.ExecutorThread.ActorSystem->GetPoolStats(poolId, poolStats, stats); + SetAggregatedCounters(PoolCounters[poolId], poolStats, stats); + } + + OnWakeup(ctx); + + ctx.Schedule(TDuration::Seconds(IntervalSec), new TEvents::TEvWakeup()); + } + + void SetAggregatedCounters(TExecutorPoolCounters& poolCounters, TExecutorPoolStats& poolStats, TVector<TExecutorThreadStats>& stats) { + // Sum all per-thread counters into the 0th element + for (ui32 idx = 1; idx < stats.size(); ++idx) { + stats[0].Aggregate(stats[idx]); + } + if (stats.size()) { + poolCounters.Set(poolStats, stats[0], stats.size() - 1); + } + } + +protected: + const ui32 IntervalSec; + NMonitoring::TDynamicCounterPtr Counters; + + TVector<TExecutorPoolCounters> PoolCounters; +}; + +} // NActors diff --git a/library/cpp/actors/helpers/selfping_actor.cpp b/library/cpp/actors/helpers/selfping_actor.cpp index 7902ff6837..f9bfaf8dc0 100644 --- a/library/cpp/actors/helpers/selfping_actor.cpp +++ b/library/cpp/actors/helpers/selfping_actor.cpp @@ -10,14 +10,14 @@ namespace NActors { namespace { -struct TEvPing: public TEventLocal<TEvPing, TEvents::THelloWorld::Ping> { - TEvPing(double timeStart) - : TimeStart(timeStart) - {} - - const double TimeStart; -}; - +struct TEvPing: public TEventLocal<TEvPing, TEvents::THelloWorld::Ping> { + TEvPing(double timeStart) + : TimeStart(timeStart) + {} + + const double TimeStart; +}; + template <class TValueType_> struct TAvgOperation { struct TValueType { @@ -70,8 +70,8 @@ private: THPTimer Timer; public: - static constexpr auto ActorActivityType() { - return SELF_PING_ACTOR; + static constexpr auto ActorActivityType() { + return SELF_PING_ACTOR; } TSelfPingActor(TDuration sendInterval, const NMonitoring::TDynamicCounters::TCounterPtr& counter, @@ -93,7 +93,7 @@ public: STFUNC(RunningState) { switch (ev->GetTypeRewrite()) { - HFunc(TEvPing, HandlePing); + HFunc(TEvPing, HandlePing); default: Y_FAIL("TSelfPingActor::RunningState: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); } @@ -146,7 +146,7 @@ public: return (ui64)d; } - void HandlePing(TEvPing::TPtr &ev, const TActorContext &ctx) + void HandlePing(TEvPing::TPtr &ev, const TActorContext &ctx) { const auto now = ctx.Now(); const double hpNow = Timer.Passed(); @@ -166,7 +166,7 @@ public: private: void SchedulePing(const TActorContext &ctx, double hpNow) const { - ctx.Schedule(SendInterval, new TEvPing(hpNow)); + ctx.Schedule(SendInterval, new TEvPing(hpNow)); } }; diff --git a/library/cpp/actors/helpers/selfping_actor.h b/library/cpp/actors/helpers/selfping_actor.h index 4217fa4293..d7d07f9fa8 100644 --- a/library/cpp/actors/helpers/selfping_actor.h +++ b/library/cpp/actors/helpers/selfping_actor.h @@ -1,6 +1,6 @@ #pragma once -#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/actor.h> #include <library/cpp/monlib/dynamic_counters/counters.h> namespace NActors { diff --git a/library/cpp/actors/helpers/selfping_actor_ut.cpp b/library/cpp/actors/helpers/selfping_actor_ut.cpp index 3643397c0b..459635fa24 100644 --- a/library/cpp/actors/helpers/selfping_actor_ut.cpp +++ b/library/cpp/actors/helpers/selfping_actor_ut.cpp @@ -1,24 +1,24 @@ #include "selfping_actor.h" #include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/actors/testlib/test_runtime.h> +#include <library/cpp/actors/testlib/test_runtime.h> namespace NActors { namespace Tests { -THolder<TTestActorRuntimeBase> CreateRuntime() { - auto runtime = MakeHolder<TTestActorRuntimeBase>(); - runtime->SetScheduledEventFilter([](auto&&, auto&&, auto&&, auto&&) { return false; }); - runtime->Initialize(); - return runtime; -} - +THolder<TTestActorRuntimeBase> CreateRuntime() { + auto runtime = MakeHolder<TTestActorRuntimeBase>(); + runtime->SetScheduledEventFilter([](auto&&, auto&&, auto&&, auto&&) { return false; }); + runtime->Initialize(); + return runtime; +} + Y_UNIT_TEST_SUITE(TSelfPingTest) { Y_UNIT_TEST(Basic) { - auto runtime = CreateRuntime(); + auto runtime = CreateRuntime(); - //const TActorId sender = runtime.AllocateEdgeActor(); + //const TActorId sender = runtime.AllocateEdgeActor(); NMonitoring::TDynamicCounters::TCounterPtr counter(new NMonitoring::TCounterForPtr()); NMonitoring::TDynamicCounters::TCounterPtr counter2(new NMonitoring::TCounterForPtr()); @@ -30,10 +30,10 @@ Y_UNIT_TEST_SUITE(TSelfPingTest) { UNIT_ASSERT_VALUES_EQUAL(counter->Val(), 0); UNIT_ASSERT_VALUES_EQUAL(counter2->Val(), 0); - const TActorId actorId = runtime->Register(actor); - Y_UNUSED(actorId); + const TActorId actorId = runtime->Register(actor); + Y_UNUSED(actorId); - //runtime.Send(new IEventHandle(actorId, sender, new TEvSelfPing::TEvPing(0.0))); + //runtime.Send(new IEventHandle(actorId, sender, new TEvSelfPing::TEvPing(0.0))); // TODO check after events are handled //Sleep(TDuration::Seconds(1)); diff --git a/library/cpp/actors/helpers/ut/ya.make b/library/cpp/actors/helpers/ut/ya.make index 933544d0e7..cba4d6d1d9 100644 --- a/library/cpp/actors/helpers/ut/ya.make +++ b/library/cpp/actors/helpers/ut/ya.make @@ -1,36 +1,36 @@ -UNITTEST_FOR(library/cpp/actors/helpers) - -OWNER( - alexvru - g:kikimr -) - -FORK_SUBTESTS() -IF (SANITIZER_TYPE) - SIZE(LARGE) - TIMEOUT(1200) - TAG(ya:fat) - SPLIT_FACTOR(20) - REQUIREMENTS( - ram:32 - ) -ELSE() - SIZE(MEDIUM) - TIMEOUT(600) - REQUIREMENTS( - ram:16 - ) -ENDIF() - - -PEERDIR( - library/cpp/actors/interconnect - library/cpp/actors/testlib - library/cpp/actors/core -) - -SRCS( - selfping_actor_ut.cpp -) - -END() +UNITTEST_FOR(library/cpp/actors/helpers) + +OWNER( + alexvru + g:kikimr +) + +FORK_SUBTESTS() +IF (SANITIZER_TYPE) + SIZE(LARGE) + TIMEOUT(1200) + TAG(ya:fat) + SPLIT_FACTOR(20) + REQUIREMENTS( + ram:32 + ) +ELSE() + SIZE(MEDIUM) + TIMEOUT(600) + REQUIREMENTS( + ram:16 + ) +ENDIF() + + +PEERDIR( + library/cpp/actors/interconnect + library/cpp/actors/testlib + library/cpp/actors/core +) + +SRCS( + selfping_actor_ut.cpp +) + +END() diff --git a/library/cpp/actors/helpers/ya.make b/library/cpp/actors/helpers/ya.make index 10cf704d82..d8771179de 100644 --- a/library/cpp/actors/helpers/ya.make +++ b/library/cpp/actors/helpers/ya.make @@ -9,7 +9,7 @@ SRCS( flow_controlled_queue.h future_callback.h mon_histogram_helper.h - selfping_actor.cpp + selfping_actor.cpp ) PEERDIR( @@ -18,8 +18,8 @@ PEERDIR( ) END() - -RECURSE_FOR_TESTS( - ut -) - + +RECURSE_FOR_TESTS( + ut +) + diff --git a/library/cpp/actors/interconnect/interconnect_tcp_server.cpp b/library/cpp/actors/interconnect/interconnect_tcp_server.cpp index fae41db0b3..b95c994598 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_server.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_server.cpp @@ -7,21 +7,21 @@ #include "interconnect_common.h" namespace NActors { - TInterconnectListenerTCP::TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket) + TInterconnectListenerTCP::TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket) : TActor(&TThis::Initial) , TInterconnectLoggingBase(Sprintf("ICListener: %s", SelfId().ToString().data())) , Address(address.c_str(), port) - , Listener( - socket - ? new NInterconnect::TStreamSocket(*socket) - : nullptr) - , ExternalSocket(!!Listener) + , Listener( + socket + ? new NInterconnect::TStreamSocket(*socket) + : nullptr) + , ExternalSocket(!!Listener) , ProxyCommonCtx(std::move(common)) - { - if (ExternalSocket) { - SetNonBlock(*Listener); - } - } + { + if (ExternalSocket) { + SetNonBlock(*Listener); + } + } TAutoPtr<IEventHandle> TInterconnectListenerTCP::AfterRegister(const TActorId& self, const TActorId& parentId) { return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); @@ -102,7 +102,7 @@ namespace NActors { ctx.Register(CreateIncomingHandshakeActor(ProxyCommonCtx, std::move(socket))); continue; } else if (-r != EAGAIN && -r != EWOULDBLOCK) { - Y_VERIFY(-r != ENFILE && -r != EMFILE && !ExternalSocket); + Y_VERIFY(-r != ENFILE && -r != EMFILE && !ExternalSocket); LOG_ERROR_IC("ICL06", "Listen failed: %s (%s)", strerror(-r), Address.ToString().data()); Listener.Reset(); PollerToken.Reset(); diff --git a/library/cpp/actors/interconnect/interconnect_tcp_server.h b/library/cpp/actors/interconnect/interconnect_tcp_server.h index aab8e97091..fc71073c2d 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_server.h +++ b/library/cpp/actors/interconnect/interconnect_tcp_server.h @@ -15,7 +15,7 @@ namespace NActors { return INTERCONNECT_COMMON; } - TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket = Nothing()); + TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket = Nothing()); int Bind(); private: @@ -45,7 +45,7 @@ namespace NActors { const NInterconnect::TAddress Address; TIntrusivePtr<NInterconnect::TStreamSocket> Listener; - const bool ExternalSocket; + const bool ExternalSocket; TPollerToken::TPtr PollerToken; TInterconnectProxyCommon::TPtr const ProxyCommonCtx; }; diff --git a/library/cpp/actors/util/local_process_key.h b/library/cpp/actors/util/local_process_key.h index 61d627bf28..172f08fc73 100644 --- a/library/cpp/actors/util/local_process_key.h +++ b/library/cpp/actors/util/local_process_key.h @@ -1,19 +1,19 @@ #pragma once -#include <util/string/builder.h> +#include <util/string/builder.h> #include <util/generic/strbuf.h> #include <util/generic/vector.h> #include <util/generic/hash.h> #include <util/generic/singleton.h> -#include <util/generic/serialized_enum.h> +#include <util/generic/serialized_enum.h> template <typename T> class TLocalProcessKeyState { template <typename U, const char* Name> friend class TLocalProcessKey; -template <typename U, typename EnumT> -friend class TEnumProcessKey; +template <typename U, typename EnumT> +friend class TEnumProcessKey; public: static TLocalProcessKeyState& GetInstance() { @@ -21,17 +21,17 @@ public: } size_t GetCount() const { - return StartIndex + Names.size(); + return StartIndex + Names.size(); } TStringBuf GetNameByIndex(size_t index) const { - if (index < StartIndex) { - return StaticNames[index]; - } else { - index -= StartIndex; - Y_ENSURE(index < Names.size()); - return Names[index]; - } + if (index < StartIndex) { + return StaticNames[index]; + } else { + index -= StartIndex; + Y_ENSURE(index < Names.size()); + return Names[index]; + } } size_t GetIndexByName(TStringBuf name) const { @@ -42,7 +42,7 @@ public: private: size_t Register(TStringBuf name) { - auto x = Map.emplace(name, Names.size()+StartIndex); + auto x = Map.emplace(name, Names.size()+StartIndex); if (x.second) { Names.emplace_back(name); } @@ -50,29 +50,29 @@ private: return x.first->second; } - size_t Register(TStringBuf name, ui32 index) { - Y_VERIFY(index < StartIndex); - auto x = Map.emplace(name, index); - Y_VERIFY(x.second || x.first->second == index); - StaticNames[index] = name; - return x.first->second; - } - + size_t Register(TStringBuf name, ui32 index) { + Y_VERIFY(index < StartIndex); + auto x = Map.emplace(name, index); + Y_VERIFY(x.second || x.first->second == index); + StaticNames[index] = name; + return x.first->second; + } + private: - static constexpr ui32 StartIndex = 2000; - - TVector<TString> FillStaticNames() { - TVector<TString> staticNames; - staticNames.reserve(StartIndex); - for (ui32 i = 0; i < StartIndex; i++) { - staticNames.push_back(TStringBuilder() << "Activity_" << i); - } - return staticNames; - } - - TVector<TString> StaticNames = FillStaticNames(); - TVector<TString> Names; - THashMap<TString, size_t> Map; + static constexpr ui32 StartIndex = 2000; + + TVector<TString> FillStaticNames() { + TVector<TString> staticNames; + staticNames.reserve(StartIndex); + for (ui32 i = 0; i < StartIndex; i++) { + staticNames.push_back(TStringBuilder() << "Activity_" << i); + } + return staticNames; + } + + TVector<TString> StaticNames = FillStaticNames(); + TVector<TString> Names; + THashMap<TString, size_t> Map; }; template <typename T, const char* Name> @@ -89,44 +89,44 @@ public: private: inline static size_t Index = TLocalProcessKeyState<T>::GetInstance().Register(Name); }; - -template <typename T, typename EnumT> -class TEnumProcessKey { -public: - static TStringBuf GetName(EnumT key) { - return TLocalProcessKeyState<T>::GetInstance().GetNameByIndex(GetIndex(key)); - } - - static size_t GetIndex(EnumT key) { - ui32 index = static_cast<ui32>(key); - if (index < TLocalProcessKeyState<T>::StartIndex) { - return index; - } - Y_VERIFY(index < Enum2Index.size()); - return Enum2Index[index]; - } - -private: - inline static TVector<size_t> RegisterAll() { - static_assert(std::is_enum<EnumT>::value, "Enum is required"); - - TVector<size_t> enum2Index; - auto names = GetEnumNames<EnumT>(); - ui32 maxId = 0; - for (const auto& [k, v] : names) { - maxId = Max(maxId, static_cast<ui32>(k)); - } - enum2Index.resize(maxId+1); - for (ui32 i = 0; i <= maxId && i < TLocalProcessKeyState<T>::StartIndex; i++) { - enum2Index[i] = i; - } - - for (const auto& [k, v] : names) { - ui32 enumId = static_cast<ui32>(k); - enum2Index[enumId] = TLocalProcessKeyState<T>::GetInstance().Register(v, enumId); - } - return enum2Index; - } - - inline static TVector<size_t> Enum2Index = RegisterAll(); -}; + +template <typename T, typename EnumT> +class TEnumProcessKey { +public: + static TStringBuf GetName(EnumT key) { + return TLocalProcessKeyState<T>::GetInstance().GetNameByIndex(GetIndex(key)); + } + + static size_t GetIndex(EnumT key) { + ui32 index = static_cast<ui32>(key); + if (index < TLocalProcessKeyState<T>::StartIndex) { + return index; + } + Y_VERIFY(index < Enum2Index.size()); + return Enum2Index[index]; + } + +private: + inline static TVector<size_t> RegisterAll() { + static_assert(std::is_enum<EnumT>::value, "Enum is required"); + + TVector<size_t> enum2Index; + auto names = GetEnumNames<EnumT>(); + ui32 maxId = 0; + for (const auto& [k, v] : names) { + maxId = Max(maxId, static_cast<ui32>(k)); + } + enum2Index.resize(maxId+1); + for (ui32 i = 0; i <= maxId && i < TLocalProcessKeyState<T>::StartIndex; i++) { + enum2Index[i] = i; + } + + for (const auto& [k, v] : names) { + ui32 enumId = static_cast<ui32>(k); + enum2Index[enumId] = TLocalProcessKeyState<T>::GetInstance().Register(v, enumId); + } + return enum2Index; + } + + inline static TVector<size_t> Enum2Index = RegisterAll(); +}; diff --git a/library/cpp/grpc/server/grpc_server.cpp b/library/cpp/grpc/server/grpc_server.cpp index 8863c80832..7437b7a8f5 100644 --- a/library/cpp/grpc/server/grpc_server.cpp +++ b/library/cpp/grpc/server/grpc_server.cpp @@ -56,22 +56,22 @@ void TGRpcServer::Start() { using grpc::ServerBuilder; using grpc::ResourceQuota; ServerBuilder builder; - auto credentials = grpc::InsecureServerCredentials(); - if (Options_.SslData) { + auto credentials = grpc::InsecureServerCredentials(); + if (Options_.SslData) { grpc::SslServerCredentialsOptions::PemKeyCertPair keycert; keycert.cert_chain = std::move(Options_.SslData->Cert); keycert.private_key = std::move(Options_.SslData->Key); grpc::SslServerCredentialsOptions sslOps; sslOps.pem_root_certs = std::move(Options_.SslData->Root); sslOps.pem_key_cert_pairs.push_back(keycert); - credentials = grpc::SslServerCredentials(sslOps); - } - if (Options_.ExternalListener) { - Options_.ExternalListener->Init(builder.experimental().AddExternalConnectionAcceptor( - ServerBuilder::experimental_type::ExternalConnectionType::FROM_FD, - credentials - )); - } else { + credentials = grpc::SslServerCredentials(sslOps); + } + if (Options_.ExternalListener) { + Options_.ExternalListener->Init(builder.experimental().AddExternalConnectionAcceptor( + ServerBuilder::experimental_type::ExternalConnectionType::FROM_FD, + credentials + )); + } else { builder.AddListeningPort(server_address, credentials); } builder.SetMaxReceiveMessageSize(Options_.MaxMessageSize); @@ -171,10 +171,10 @@ void TGRpcServer::Start() { })); } } - - if (Options_.ExternalListener) { - Options_.ExternalListener->Start(); - } + + if (Options_.ExternalListener) { + Options_.ExternalListener->Start(); + } } void TGRpcServer::Stop() { @@ -223,10 +223,10 @@ void TGRpcServer::Stop() { } Ts.clear(); - - if (Options_.ExternalListener) { - Options_.ExternalListener->Stop(); - } + + if (Options_.ExternalListener) { + Options_.ExternalListener->Stop(); + } } ui16 TGRpcServer::GetPort() const { diff --git a/library/cpp/grpc/server/grpc_server.h b/library/cpp/grpc/server/grpc_server.h index 9cb55e3154..d6814a90a0 100644 --- a/library/cpp/grpc/server/grpc_server.h +++ b/library/cpp/grpc/server/grpc_server.h @@ -27,15 +27,15 @@ struct TSslData { TString Root; }; -struct IExternalListener - : public TThrRefBase -{ - using TPtr = TIntrusivePtr<IExternalListener>; - virtual void Init(std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> acceptor) = 0; - virtual void Start() = 0; - virtual void Stop() = 0; -}; - +struct IExternalListener + : public TThrRefBase +{ + using TPtr = TIntrusivePtr<IExternalListener>; + virtual void Init(std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> acceptor) = 0; + virtual void Start() = 0; + virtual void Stop() = 0; +}; + //! Server's options. struct TServerOptions { #define DECLARE_FIELD(name, type, default) \ @@ -93,8 +93,8 @@ struct TServerOptions { //! Custom configurator for ServerBuilder. DECLARE_FIELD(ServerBuilderMutator, std::function<void(grpc::ServerBuilder&)>, [](grpc::ServerBuilder&){}); - DECLARE_FIELD(ExternalListener, IExternalListener::TPtr, nullptr); - + DECLARE_FIELD(ExternalListener, IExternalListener::TPtr, nullptr); + //! Logger which will be used to write logs about requests handling (iff appropriate log level is enabled). DECLARE_FIELD(Logger, TLoggerPtr, nullptr); diff --git a/ydb/core/actorlib_impl/actor_activity_ut.cpp b/ydb/core/actorlib_impl/actor_activity_ut.cpp index 2374b99eb6..119aa53978 100644 --- a/ydb/core/actorlib_impl/actor_activity_ut.cpp +++ b/ydb/core/actorlib_impl/actor_activity_ut.cpp @@ -1,38 +1,38 @@ #include <ydb/core/actorlib_impl/defs.h> - - -#include <library/cpp/testing/unittest/registar.h> + + +#include <library/cpp/testing/unittest/registar.h> #include <ydb/core/testlib/basics/runtime.h> #include <ydb/core/testlib/basics/appdata.h> - - -#include <library/cpp/actors/core/actor.h> + + +#include <library/cpp/actors/core/actor.h> #include <ydb/core/protos/services.pb.h> - -namespace NActors { - -class TTestActor : public TActor<TTestActor> { -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { - return NKikimrServices::TActivity::ASYNC_DESTROYER; - } - - TTestActor() - : TActor(nullptr) - { } -}; - -} // namespace NActors - -Y_UNIT_TEST_SUITE(TActorActivity) { - using namespace NActors; - - Y_UNIT_TEST(Basic) { - TAutoPtr<IActor> actor = new TTestActor(); - UNIT_ASSERT_VALUES_EQUAL(actor->GetActivityType(), static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER)); - - UNIT_ASSERT((TLocalProcessKeyState<TActorActivityTag>::GetInstance().GetIndexByName("ASYNC_DESTROYER") == static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER))); - - UNIT_ASSERT((TLocalProcessKeyState<TActorActivityTag>::GetInstance().GetNameByIndex(static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER)) == "ASYNC_DESTROYER")); - } -} + +namespace NActors { + +class TTestActor : public TActor<TTestActor> { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::ASYNC_DESTROYER; + } + + TTestActor() + : TActor(nullptr) + { } +}; + +} // namespace NActors + +Y_UNIT_TEST_SUITE(TActorActivity) { + using namespace NActors; + + Y_UNIT_TEST(Basic) { + TAutoPtr<IActor> actor = new TTestActor(); + UNIT_ASSERT_VALUES_EQUAL(actor->GetActivityType(), static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER)); + + UNIT_ASSERT((TLocalProcessKeyState<TActorActivityTag>::GetInstance().GetIndexByName("ASYNC_DESTROYER") == static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER))); + + UNIT_ASSERT((TLocalProcessKeyState<TActorActivityTag>::GetInstance().GetNameByIndex(static_cast<ui32>(NKikimrServices::TActivity::ASYNC_DESTROYER)) == "ASYNC_DESTROYER")); + } +} diff --git a/ydb/core/actorlib_impl/destruct_actor.h b/ydb/core/actorlib_impl/destruct_actor.h index 9f3b05c8e5..adc236911d 100644 --- a/ydb/core/actorlib_impl/destruct_actor.h +++ b/ydb/core/actorlib_impl/destruct_actor.h @@ -9,7 +9,7 @@ namespace NActors { class TDestructActor: public TActor<TDestructActor> { public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::INTERCONNECT_DESTRUCT_ACTOR; } diff --git a/ydb/core/actorlib_impl/long_timer.cpp b/ydb/core/actorlib_impl/long_timer.cpp index 06e6e18221..e3b139579f 100644 --- a/ydb/core/actorlib_impl/long_timer.cpp +++ b/ydb/core/actorlib_impl/long_timer.cpp @@ -37,7 +37,7 @@ class TLongTimer : public TActor<TLongTimer> { ctx.Schedule(TDuration::Seconds(ThresholdSec), new TEvents::TEvWakeup()); } public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::ACTORLIB_LONG_TIMER; } diff --git a/ydb/core/actorlib_impl/mad_squirrel.cpp b/ydb/core/actorlib_impl/mad_squirrel.cpp index ae72763fc6..3f9629d099 100644 --- a/ydb/core/actorlib_impl/mad_squirrel.cpp +++ b/ydb/core/actorlib_impl/mad_squirrel.cpp @@ -10,7 +10,7 @@ class TMadSquirrel : public TActor<TMadSquirrel> { } public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return ACTORLIB_COMMON; } diff --git a/ydb/core/actorlib_impl/ut/ya.make b/ydb/core/actorlib_impl/ut/ya.make index b4932c22e0..f76d57807a 100644 --- a/ydb/core/actorlib_impl/ut/ya.make +++ b/ydb/core/actorlib_impl/ut/ya.make @@ -28,7 +28,7 @@ PEERDIR( ) SRCS( - actor_activity_ut.cpp + actor_activity_ut.cpp actor_bootstrapped_ut.cpp actor_tracker_ut.cpp test_interconnect_ut.cpp diff --git a/ydb/core/base/actor_activity_names.cpp b/ydb/core/base/actor_activity_names.cpp index 0f9b38a3b2..d713a6ddd8 100644 --- a/ydb/core/base/actor_activity_names.cpp +++ b/ydb/core/base/actor_activity_names.cpp @@ -1,30 +1,30 @@ -#include <util/generic/serialized_enum.h> -#include <util/generic/singleton.h> +#include <util/generic/serialized_enum.h> +#include <util/generic/singleton.h> #include <ydb/core/protos/services.pb.h> - -namespace NEnumSerializationRuntime { - -using TType = NDetail::TSelectEnumRepresentationType<NKikimrServices::TActivity::EType>::TType; - -class TStorageHolder { -public: - TStorageHolder() { - for (int i = 0; i < NKikimrServices::TActivity::EType_ARRAYSIZE; i++) { - if (NKikimrServices::TActivity::EType_IsValid(i)) { - NKikimrServices::TActivity::EType enumType = static_cast<NKikimrServices::TActivity::EType>(i); - TString name = NKikimrServices::TActivity::EType_Name(enumType); - Storage.insert({enumType, name}); - } - } - } - - TMap<TType, TString> Storage; -}; - -template<> -TMappedDictView<NKikimrServices::TActivity::EType, TString> GetEnumNamesImpl() { - auto& storage = Singleton<TStorageHolder>()->Storage; - return TMappedDictView<NKikimrServices::TActivity::EType, TString>(storage); -} - -} // NEnumSerializationRuntime + +namespace NEnumSerializationRuntime { + +using TType = NDetail::TSelectEnumRepresentationType<NKikimrServices::TActivity::EType>::TType; + +class TStorageHolder { +public: + TStorageHolder() { + for (int i = 0; i < NKikimrServices::TActivity::EType_ARRAYSIZE; i++) { + if (NKikimrServices::TActivity::EType_IsValid(i)) { + NKikimrServices::TActivity::EType enumType = static_cast<NKikimrServices::TActivity::EType>(i); + TString name = NKikimrServices::TActivity::EType_Name(enumType); + Storage.insert({enumType, name}); + } + } + } + + TMap<TType, TString> Storage; +}; + +template<> +TMappedDictView<NKikimrServices::TActivity::EType, TString> GetEnumNamesImpl() { + auto& storage = Singleton<TStorageHolder>()->Storage; + return TMappedDictView<NKikimrServices::TActivity::EType, TString>(storage); +} + +} // NEnumSerializationRuntime diff --git a/ydb/core/base/pool_stats_collector.cpp b/ydb/core/base/pool_stats_collector.cpp index 048f03457b..7dcbf28954 100644 --- a/ydb/core/base/pool_stats_collector.cpp +++ b/ydb/core/base/pool_stats_collector.cpp @@ -7,22 +7,22 @@ #include <ydb/library/yql/minikql/aligned_page_pool.h> #include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/helpers/pool_stats_collector.h> +#include <library/cpp/actors/helpers/pool_stats_collector.h> namespace NKikimr { // Periodically collects stats from executor threads and exposes them as mon counters -class TStatsCollectingActor : public NActors::TStatsCollectingActor { -public: - TStatsCollectingActor( - ui32 intervalSec, - const TActorSystemSetup& setup, - NMonitoring::TDynamicCounterPtr counters) - : NActors::TStatsCollectingActor(intervalSec, setup, GetServiceCounters(counters, "utils")) - { - MiniKQLPoolStats.Init(Counters.Get()); - } - +class TStatsCollectingActor : public NActors::TStatsCollectingActor { +public: + TStatsCollectingActor( + ui32 intervalSec, + const TActorSystemSetup& setup, + NMonitoring::TDynamicCounterPtr counters) + : NActors::TStatsCollectingActor(intervalSec, setup, GetServiceCounters(counters, "utils")) + { + MiniKQLPoolStats.Init(Counters.Get()); + } + private: class TMiniKQLPoolStats { public: @@ -40,14 +40,14 @@ private: NMonitoring::TDynamicCounters::TCounterPtr TotalBytes; }; - void OnWakeup(const TActorContext &ctx) override { + void OnWakeup(const TActorContext &ctx) override { MiniKQLPoolStats.Update(); TVector<std::tuple<TString, double, ui32>> pools; for (const auto& pool : PoolCounters) { pools.emplace_back(pool.Name, pool.Usage, pool.Threads); } - + ctx.Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(ctx.SelfID.NodeId()), new NNodeWhiteboard::TEvWhiteboard::TEvSystemStateUpdate(pools)); } diff --git a/ydb/core/base/ya.make b/ydb/core/base/ya.make index c9a89eac35..83db5825c3 100644 --- a/ydb/core/base/ya.make +++ b/ydb/core/base/ya.make @@ -14,7 +14,7 @@ IF (KIKIMR_DEFAULT_SHARDED_COMPACTION) ENDIF() SRCS( - actor_activity_names.cpp + actor_activity_names.cpp appdata.h appdata.cpp board_lookup.cpp diff --git a/ydb/core/blobstorage/testload/test_load_keyvalue_write.cpp b/ydb/core/blobstorage/testload/test_load_keyvalue_write.cpp index 812dbc5556..00e31ff18d 100644 --- a/ydb/core/blobstorage/testload/test_load_keyvalue_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_keyvalue_write.cpp @@ -133,7 +133,7 @@ class TKeyValueWriterTestLoadActor : public TActorBootstrapped<TKeyValueWriterTe NMonitoring::TPercentileTrackerLg<6, 5, 15> ResponseTimes; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE; } diff --git a/ydb/core/blobstorage/testload/test_load_kqp.cpp b/ydb/core/blobstorage/testload/test_load_kqp.cpp index a7d3ba824a..fec4fd7b2c 100644 --- a/ydb/core/blobstorage/testload/test_load_kqp.cpp +++ b/ydb/core/blobstorage/testload/test_load_kqp.cpp @@ -87,7 +87,7 @@ class TKqpWriterTestLoadActor : public TActorBootstrapped<TKqpWriterTestLoadActo TString PreparedSelectQuery; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_TEST_WORKLOAD; } diff --git a/ydb/core/blobstorage/testload/test_load_memory.cpp b/ydb/core/blobstorage/testload/test_load_memory.cpp index f2b8646d5d..9da565186a 100644 --- a/ydb/core/blobstorage/testload/test_load_memory.cpp +++ b/ydb/core/blobstorage/testload/test_load_memory.cpp @@ -26,7 +26,7 @@ class TMemoryTestLoadActor : public TActorBootstrapped<TMemoryTestLoadActor> { ui64 AllocatedSize = 0; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE; } diff --git a/ydb/core/blobstorage/testload/test_load_pdisk_log.cpp b/ydb/core/blobstorage/testload/test_load_pdisk_log.cpp index 56c6a9df77..33fc363c93 100644 --- a/ydb/core/blobstorage/testload/test_load_pdisk_log.cpp +++ b/ydb/core/blobstorage/testload/test_load_pdisk_log.cpp @@ -301,7 +301,7 @@ class TPDiskLogWriterTestLoadActor : public TActorBootstrapped<TPDiskLogWriterTe NMonitoring::TPercentileTrackerLg<6, 5, 15> LogResponseTimes; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_LOAD_PDISK_LOG_WRITE; } diff --git a/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp b/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp index ccbd3a82b3..3b446a74c2 100644 --- a/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp +++ b/ydb/core/blobstorage/testload/test_load_pdisk_read.cpp @@ -117,7 +117,7 @@ class TPDiskReaderTestLoadActor : public TActorBootstrapped<TPDiskReaderTestLoad TString ErrorReason; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_LOAD_PDISK_READ; } diff --git a/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp b/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp index 330caf7e36..8ea16e014c 100644 --- a/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp +++ b/ydb/core/blobstorage/testload/test_load_pdisk_write.cpp @@ -127,7 +127,7 @@ class TPDiskWriterTestLoadActor : public TActorBootstrapped<TPDiskWriterTestLoad TMap<double, TIntrusivePtr<NMonitoring::TCounterForPtr>> DevicePercentiles; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_LOAD_PDISK_WRITE; } diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugedelete.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugedelete.cpp index 1407ab0d24..3c5e904a27 100644 --- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugedelete.cpp +++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugedelete.cpp @@ -10,7 +10,7 @@ namespace NKikimr { const TIntrusivePtr<TDelayedHugeBlobDeleterInfo> Info; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_DELAYED_HUGE_BLOB_DELETER; } diff --git a/ydb/core/blobstorage/vdisk/hulldb/hulldb_bulksstmngr.cpp b/ydb/core/blobstorage/vdisk/hulldb/hulldb_bulksstmngr.cpp index c307f7d1be..3cc7d07a19 100644 --- a/ydb/core/blobstorage/vdisk/hulldb/hulldb_bulksstmngr.cpp +++ b/ydb/core/blobstorage/vdisk/hulldb/hulldb_bulksstmngr.cpp @@ -32,7 +32,7 @@ namespace NKikimr { TActiveActors ActiveActors; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_BULK_SST_LOADER; } diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_replbroker.cpp b/ydb/core/blobstorage/vdisk/repl/blobstorage_replbroker.cpp index be7c83f410..543f0a537d 100644 --- a/ydb/core/blobstorage/vdisk/repl/blobstorage_replbroker.cpp +++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_replbroker.cpp @@ -38,7 +38,7 @@ namespace NKikimr { TReplMemTokenId NextMemToken = 1; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_REPL_BROKER; } diff --git a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer_scheduler.cpp b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer_scheduler.cpp index 77807a1239..92ae94e89f 100644 --- a/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer_scheduler.cpp +++ b/ydb/core/blobstorage/vdisk/syncer/blobstorage_syncer_scheduler.cpp @@ -163,7 +163,7 @@ namespace NKikimr { ) public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BS_SYNCER_COMMITTER_PROXY; } diff --git a/ydb/core/client/server/msgbus_servicereq.h b/ydb/core/client/server/msgbus_servicereq.h index ff5a2af9e4..704a231dd1 100644 --- a/ydb/core/client/server/msgbus_servicereq.h +++ b/ydb/core/client/server/msgbus_servicereq.h @@ -50,7 +50,7 @@ public: } } - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return Activity; } }; diff --git a/ydb/core/client/server/msgbus_tabletreq.h b/ydb/core/client/server/msgbus_tabletreq.h index 817e1ca985..f173ae9097 100644 --- a/ydb/core/client/server/msgbus_tabletreq.h +++ b/ydb/core/client/server/msgbus_tabletreq.h @@ -134,7 +134,7 @@ public: return std::pair<ui64, TAutoPtr<IEventBase>>(TabletID, static_cast<TDerived *>(this)->MakeReq(ctx)); } - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return Activity; } }; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 3371e20d8a..819c1478d1 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -159,8 +159,8 @@ #include <library/cpp/actors/core/scheduler_basic.h> #include <library/cpp/actors/core/io_dispatcher.h> #include <library/cpp/actors/dnsresolver/dnsresolver.h> -#include <library/cpp/actors/helpers/selfping_actor.h> -#include <library/cpp/actors/http/http_proxy.h> +#include <library/cpp/actors/helpers/selfping_actor.h> +#include <library/cpp/actors/http/http_proxy.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/interconnect_mon.h> #include <library/cpp/actors/interconnect/interconnect_tcp_proxy.h> @@ -516,7 +516,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters = appData->Counters; setup->NodeId = NodeId; - setup->MaxActivityType = GetActivityTypeCount(); + setup->MaxActivityType = GetActivityTypeCount(); setup->CpuManager = CreateCpuManagerConfig(systemConfig, setup->MaxActivityType); for (ui32 poolId = 0; poolId != setup->GetExecutorsCount(); ++poolId) { const auto &execConfig = systemConfig.GetExecutor(poolId); @@ -2227,18 +2227,18 @@ void TSchemeBoardMonitoringInitializer::InitializeServices(TActorSystemSetup* se TActorSetupCmd(CreateSchemeBoardMonitoring(), TMailboxType::HTSwap, appData->UserPoolId))); } -TYqlLogsInitializer::TYqlLogsInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{ -} - -void TYqlLogsInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { - setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( - MakeYqlLogsUpdaterId(), - TActorSetupCmd(CreateYqlLogsUpdater(Config.GetLogConfig()), TMailboxType::HTSwap, appData->UserPoolId) - )); -} - +TYqlLogsInitializer::TYqlLogsInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TYqlLogsInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + MakeYqlLogsUpdaterId(), + TActorSetupCmd(CreateYqlLogsUpdater(Config.GetLogConfig()), TMailboxType::HTSwap, appData->UserPoolId) + )); +} + THealthCheckInitializer::THealthCheckInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index b2c58e96ab..407ce1bb7b 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -472,13 +472,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TYqlLogsInitializer : public IKikimrServicesInitializer { -public: - TYqlLogsInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - +class TYqlLogsInitializer : public IKikimrServicesInitializer { +public: + TYqlLogsInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class THealthCheckInitializer : public IKikimrServicesInitializer { public: THealthCheckInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index e20835f351..a4f74aa4e0 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -539,7 +539,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) { names["logstore"] = &hasLogStore; bool hasAuth = services.empty(); names["auth"] = &hasAuth; - + std::unordered_set<TString> enabled; for (const auto& name : services) { enabled.insert(name); @@ -1312,9 +1312,9 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers } if (serviceMask.EnableKqp || serviceMask.EnableYandexQuery) { - sil->AddServiceInitializer(new TYqlLogsInitializer(runConfig)); - } - + sil->AddServiceInitializer(new TYqlLogsInitializer(runConfig)); + } + if (serviceMask.EnableYandexQuery && runConfig.AppConfig.GetYandexQueryConfig().GetEnabled()) { YqSharedResources = NYq::CreateYqSharedResources(runConfig.AppConfig.GetYandexQueryConfig(), ModuleFactories->YdbCredentialProviderFactory, Counters->GetSubgroup("counters", "yq")); diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp index 23de97a31a..3dd053e372 100644 --- a/ydb/core/engine/mkql_proto.cpp +++ b/ydb/core/engine/mkql_proto.cpp @@ -67,11 +67,11 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c case TType::EKind::Void: return NUdf::TUnboxedValuePod::Void(); - case TType::EKind::Null: - case TType::EKind::EmptyList: - case TType::EKind::EmptyDict: - return NUdf::TUnboxedValue(); - + case TType::EKind::Null: + case TType::EKind::EmptyList: + case TType::EKind::EmptyDict: + return NUdf::TUnboxedValue(); + case TType::EKind::Data: return HandleKindDataImport(type, value); @@ -140,20 +140,20 @@ NUdf::TUnboxedValue ImportValueFromProto(TType* type, const Ydb::Value& value, c return dictBuilder->Build(); } - case TType::EKind::Variant: { - auto variantType = static_cast<TVariantType*>(type); - auto index = value.variant_index(); - auto unboxedValue = ImportValueFromProto(variantType->GetAlternativeType(index), value, factory); - auto res = factory.CreateVariantHolder(std::move(unboxedValue.Release()), index); - return std::move(res); - } - - case TType::EKind::Tagged: { - auto taggedType = static_cast<TTaggedType*>(type); - auto unboxedValue = ImportValueFromProto(taggedType->GetBaseType(), value, factory); - return std::move(unboxedValue); - } - + case TType::EKind::Variant: { + auto variantType = static_cast<TVariantType*>(type); + auto index = value.variant_index(); + auto unboxedValue = ImportValueFromProto(variantType->GetAlternativeType(index), value, factory); + auto res = factory.CreateVariantHolder(std::move(unboxedValue.Release()), index); + return std::move(res); + } + + case TType::EKind::Tagged: { + auto taggedType = static_cast<TTaggedType*>(type); + auto unboxedValue = ImportValueFromProto(taggedType->GetBaseType(), value, factory); + return std::move(unboxedValue); + } + default: MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); } diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 9ce62092ba..44b25c4a5f 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -137,7 +137,7 @@ struct TRpcServices { EvLongTxRead, EvExplainYqlScript, EvImportData, - EvAnalyticsReserved, + EvAnalyticsReserved, EvDataStreamsCreateStream, EvDataStreamsDeleteStream, EvDataStreamsDescribeStream, diff --git a/ydb/core/kqp/compile/kqp_compile.cpp b/ydb/core/kqp/compile/kqp_compile.cpp index aa00dda190..c7d305ab46 100644 --- a/ydb/core/kqp/compile/kqp_compile.cpp +++ b/ydb/core/kqp/compile/kqp_compile.cpp @@ -569,7 +569,7 @@ private: } stageProto.SetProgramAst(KqpExprToPrettyString(stage.Program(), ctx)); - stageProto.SetStageGuid(NDq::TDqStageSettings::Parse(stage).Id); + stageProto.SetStageGuid(NDq::TDqStageSettings::Parse(stage).Id); } void CompileTransaction(const TKqpPhysicalTx& tx, NKqpProto::TKqpPhyTx& txProto, TExprContext& ctx) { diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp index d243a60da0..780f865e9e 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp @@ -1,33 +1,33 @@ -#include "kqp_compute_actor.h" - +#include "kqp_compute_actor.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/kqp/runtime/kqp_compute.h> #include <ydb/core/kqp/runtime/kqp_read_table.h> - -namespace NKikimr { -namespace NMiniKQL { - -using TCallableActorBuilderFunc = std::function< - IComputationNode*( - TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx)>; - -TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* computeCtx) { - MKQL_ENSURE_S(computeCtx); - - auto computeFactory = GetKqpBaseComputeFactory(computeCtx); - - return [computeFactory, computeCtx] - (TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { - if (auto compute = computeFactory(callable, ctx)) { - return compute; - } - + +namespace NKikimr { +namespace NMiniKQL { + +using TCallableActorBuilderFunc = std::function< + IComputationNode*( + TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx)>; + +TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* computeCtx) { + MKQL_ENSURE_S(computeCtx); + + auto computeFactory = GetKqpBaseComputeFactory(computeCtx); + + return [computeFactory, computeCtx] + (TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* { + if (auto compute = computeFactory(callable, ctx)) { + return compute; + } + auto name = callable.GetType()->GetName(); if (name == "KqpWideReadTable"sv) { return WrapKqpScanWideReadTable(callable, ctx, *computeCtx); - } - + } + if (name == "KqpWideReadTableRanges"sv) { return WrapKqpScanWideReadTableRanges(callable, ctx, *computeCtx); } @@ -37,10 +37,10 @@ TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* comput return WrapKqpEnsure(callable, ctx); } - return nullptr; - }; -} - -} // namespace NMiniKQL -} // namespace NKikimr - + return nullptr; + }; +} + +} // namespace NMiniKQL +} // namespace NKikimr + diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.h b/ydb/core/kqp/compute_actor/kqp_compute_actor.h index 8f034ffc2c..02b7b98029 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <ydb/core/kqp/counters/kqp_counters.h> #include <ydb/core/kqp/kqp_compute.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - -namespace NKikimr { -namespace NMiniKQL { - + +namespace NKikimr { +namespace NMiniKQL { + class TKqpScanComputeContext; -TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* computeCtx); - -} // namespace NMiniKQL - -namespace NKqp { - +TComputationNodeFactory GetKqpActorComputeFactory(TKqpScanComputeContext* computeCtx); + +} // namespace NMiniKQL + +namespace NKqp { + IActor* CreateKqpComputeActor(const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task, NYql::NDq::IDqSourceActorFactory::TPtr sourceActorFactory, NYql::NDq::IDqSinkActorFactory::TPtr sinkActorFactory, const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits); - + IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId, NYql::NDqProto::TDqTask&& task, NYql::NDq::IDqSourceActorFactory::TPtr sourceActorFactory, NYql::NDq::IDqSinkActorFactory::TPtr sinkActorFactory, const NYql::NDq::TComputeRuntimeSettings& settings, const NYql::NDq::TComputeMemoryLimits& memoryLimits, TIntrusivePtr<TKqpCounters> counters); -} // namespace NKqp -} // namespace NKikimr +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp index 507adc6f08..f8e3cd0c77 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp @@ -12,7 +12,7 @@ #include <ydb/core/tx/datashard/range_ops.h> #include <ydb/core/tx/scheme_cache/scheme_cache.h> #include <ydb/core/grpc_services/local_rate_limiter.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -20,19 +20,19 @@ #include <util/generic/deque.h> -namespace NKikimr { -namespace NKqp { - -namespace { - +namespace NKikimr { +namespace NKqp { + +namespace { + using namespace NYql; using namespace NYql::NDq; bool IsDebugLogEnabled(const TActorSystem* actorSystem, NActors::NLog::EComponent component) { - auto* settings = actorSystem->LoggerSettings(); + auto* settings = actorSystem->LoggerSettings(); return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, component); -} - +} + TString DebugPrintRanges(TConstArrayRef<NScheme::TTypeId> types, const TSmallVec<TSerializedTableRange>& ranges) { @@ -66,23 +66,23 @@ class TKqpScanComputeActor : public TDqComputeActorBase<TKqpScanComputeActor> { struct TEvRetryShard : public TEventLocal<TEvRetryShard, EvRetryShard> {}; }; -public: - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::KQP_SCAN_COMPUTE_ACTOR; - } - + } + TKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, TIntrusivePtr<TKqpCounters> counters) : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits) , ComputeCtx(settings.StatsMode) - , Snapshot(snapshot) + , Snapshot(snapshot) , Counters(counters) { YQL_ENSURE(GetTask().GetMeta().UnpackTo(&Meta), "Invalid task meta: " << GetTask().GetMeta().DebugString()); YQL_ENSURE(!Meta.GetReads().empty()); YQL_ENSURE(Meta.GetTable().GetTableKind() != (ui32)ETableKind::SysView); - + KeyColumnTypes.assign(Meta.GetKeyColumnTypes().begin(), Meta.GetKeyColumnTypes().end()); } @@ -763,7 +763,7 @@ private: } TBase::HandleExecuteBase(ev); } - + void HandleExecute(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { auto nodeId = ev->Get()->NodeId; CA_LOG_N("Disconnected node " << nodeId); @@ -781,20 +781,20 @@ private: void StartTableScan() { YQL_ENSURE(!Shards.empty()); - + auto& state = Shards.front(); - + YQL_ENSURE(state.State == EShardState::Initial); state.State = EShardState::Starting; state.Generation = ++LastGeneration; state.ActorId = {}; - + CA_LOG_D("StartTableScan: '" << ScanData->TablePath << "', shardId: " << state.TabletId << ", gen: " << state.Generation << ", ranges: " << DebugPrintRanges(KeyColumnTypes, GetScanRanges(state))); SendStartScanRequest(state, state.Generation); } - + void SendScanDataAck(TShardState& state, ui64 freeSpace) { CA_LOG_D("Send EvScanDataAck to " << state.ActorId << ", freeSpace: " << freeSpace << ", gen: " << state.Generation); ui32 flags = IEventHandle::FlagTrackDelivery; @@ -803,7 +803,7 @@ private: } Send(state.ActorId, new TEvKqpCompute::TEvScanDataAck(freeSpace, state.Generation), flags); } - + void SendStartScanRequest(TShardState& state, ui32 gen) { YQL_ENSURE(state.State == EShardState::Starting); @@ -812,7 +812,7 @@ private: for (auto& column: ScanData->GetColumns()) { ev->Record.AddColumnTags(column.Tag); ev->Record.AddColumnTypes(column.Type); - } + } ev->Record.MutableSkipNullKeys()->CopyFrom(Meta.GetSkipNullKeys()); auto ranges = GetScanRanges(state); @@ -824,16 +824,16 @@ private: range.Serialize(*newRange); } - ev->Record.MutableSnapshot()->CopyFrom(Snapshot); - if (RuntimeSettings.Timeout) { - ev->Record.SetTimeoutMs(RuntimeSettings.Timeout.Get()->MilliSeconds()); - } - ev->Record.SetStatsMode(RuntimeSettings.StatsMode); + ev->Record.MutableSnapshot()->CopyFrom(Snapshot); + if (RuntimeSettings.Timeout) { + ev->Record.SetTimeoutMs(RuntimeSettings.Timeout.Get()->MilliSeconds()); + } + ev->Record.SetStatsMode(RuntimeSettings.StatsMode); ev->Record.SetScanId(0); ev->Record.SetTxId(std::get<ui64>(TxId)); ev->Record.SetTablePath(ScanData->TablePath); ev->Record.SetSchemaVersion(ScanData->TableId.SchemaVersion); - + ev->Record.SetGeneration(gen); ev->Record.SetReverse(Meta.GetReverse()); @@ -858,9 +858,9 @@ private: << ", range: " << DebugPrintRanges(KeyColumnTypes, GetScanRanges(state))); Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), state.TabletId, !subscribed), - IEventHandle::FlagTrackDelivery); - } - + IEventHandle::FlagTrackDelivery); + } + const TSmallVec<TSerializedTableRange> GetScanRanges(const TShardState& state) const { // No any data read previously, return all ranges if (!LastKey.DataSize()) { @@ -959,7 +959,7 @@ private: struct TScanFreeSpace : public IDestructable { ui64 FreeSpace = 0; }; - + THolder<IDestructable> GetSourcesState() override { if (ScanData) { auto state = MakeHolder<TScanFreeSpace>(); @@ -970,31 +970,31 @@ private: } return nullptr; } - + void PollSources(THolder<IDestructable> prev) override { if (!prev || !ScanData || Shards.empty()) { return; - } - + } + auto& state = Shards.front(); ui64 freeSpace = GetMemoryLimits().ScanBufferSize > ScanData->GetStoredBytes() ? GetMemoryLimits().ScanBufferSize - ScanData->GetStoredBytes() : 0ul; ui64 prevFreeSpace = static_cast<TScanFreeSpace*>(prev.Get())->FreeSpace; - + CA_LOG_T("Scan over tablet " << state.TabletId << " finished: " << ScanData->IsFinished() << ", prevFreeSpace: " << prevFreeSpace << ", freeSpace: " << freeSpace << ", peer: " << state.ActorId); - + if (!ScanData->IsFinished() && state.State != EShardState::PostRunning && prevFreeSpace < freeSpace && state.ActorId) { CA_LOG_T("[poll] Send EvScanDataAck to " << state.ActorId << ", gen: " << state.Generation << ", freeSpace: " << freeSpace); SendScanDataAck(state, freeSpace); - } + } } - + void TerminateSources(const TString& message, bool success) override { if (!ScanData || Shards.empty()) { return; @@ -1005,13 +1005,13 @@ private: if (state.ActorId) { CA_LOG(prio, "Send abort execution event to scan over tablet: " << state.TabletId << ", table: " << ScanData->TablePath << ", scan actor: " << state.ActorId << ", message: " << message); - + Send(state.ActorId, new TEvKqp::TEvAbortExecution( success ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::ABORTED, message)); } else { CA_LOG(prio, "Table: " << ScanData->TablePath << ", scan has not been started yet"); } - } + } void PassAway() override { Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); @@ -1128,17 +1128,17 @@ private: ui32 LastGeneration = 0; TSet<ui64> AffectedShards; THashSet<ui32> TrackingNodes; -}; - +}; + } // anonymous namespace - + IActor* CreateKqpScanComputeActor(const NKikimrKqp::TKqpSnapshot& snapshot, const TActorId& executerId, ui64 txId, NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, TIntrusivePtr<TKqpCounters> counters) -{ +{ return new TKqpScanComputeActor(snapshot, executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits, counters); -} - -} // namespace NKqp -} // namespace NKikimr +} + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/compute_actor/ya.make b/ydb/core/kqp/compute_actor/ya.make index 9c9b747c62..3a39e3c19a 100644 --- a/ydb/core/kqp/compute_actor/ya.make +++ b/ydb/core/kqp/compute_actor/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER( - spuchin - g:kikimr -) - -SRCS( - kqp_compute_actor.cpp +LIBRARY() + +OWNER( + spuchin + g:kikimr +) + +SRCS( + kqp_compute_actor.cpp kqp_pure_compute_actor.cpp kqp_scan_compute_actor.cpp -) - -PEERDIR( +) + +PEERDIR( ydb/core/actorlib_impl ydb/core/base ydb/core/kqp/runtime @@ -19,8 +19,8 @@ PEERDIR( ydb/core/tx/scheme_cache ydb/library/yql/dq/actors/compute ydb/library/yql/public/issue -) - +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/core/kqp/executer/kqp_data_executer.cpp b/ydb/core/kqp/executer/kqp_data_executer.cpp index 8bd740a835..003d4ffa20 100644 --- a/ydb/core/kqp/executer/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer/kqp_data_executer.cpp @@ -1229,7 +1229,7 @@ private: limits.ChannelBufferSize = 50_MB; limits.MkqlLightProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(500_MB, Request.MkqlMemoryLimit) : 500_MB; limits.MkqlHeavyProgramMemoryLimit = Request.MkqlMemoryLimit > 0 ? std::min(2_GB, Request.MkqlMemoryLimit) : 2_GB; - limits.AllocateMemoryFn = [TxId = TxId](auto /* txId */, ui64 taskId, ui64 memory) { + limits.AllocateMemoryFn = [TxId = TxId](auto /* txId */, ui64 taskId, ui64 memory) { LOG_E("Data query task cannot allocate additional memory during executing." << " Task: " << taskId << ", memory: " << memory); return false; diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp index 6d8d14fd45..27cf296a03 100644 --- a/ydb/core/kqp/kqp_compile_actor.cpp +++ b/ydb/core/kqp/kqp_compile_actor.cpp @@ -8,7 +8,7 @@ #include <ydb/core/kqp/host/kqp_host.h> #include <ydb/library/yql/utils/actor_log/log.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/json/json_writer.h> @@ -33,7 +33,7 @@ static const TString YqlName = "CompileActor"; using namespace NKikimrConfig; using namespace NThreading; using namespace NYql; -using namespace NYql::NDq; +using namespace NYql::NDq; class TKqpCompileActor : public TActorBootstrapped<TKqpCompileActor> { public: @@ -83,7 +83,7 @@ public: TimeoutTimerActorId = CreateLongTimer(ctx, CompilationTimeout, new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup())); - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); TKqpRequestCounters::TPtr counters = new TKqpRequestCounters; counters->Counters = Counters; @@ -240,7 +240,7 @@ private: void Handle(TEvKqp::TEvContinueProcess::TPtr &ev, const TActorContext &ctx) { Y_ENSURE(!ev->Get()->QueryId); - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); if (!ev->Get()->Finished) { NCpuTime::TCpuTimer timer(CompileCpuTime); @@ -322,7 +322,7 @@ private: void HandleRecompile(TEvKqp::TEvContinueProcess::TPtr &ev, const TActorContext &ctx) { Y_ENSURE(!ev->Get()->QueryId); - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, YqlName, ""); if (!ev->Get()->Finished) { NCpuTime::TCpuTimer timer(CompileCpuTime); diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index 08ab4e0eb2..cccb8a51e8 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -16,7 +16,7 @@ #include <ydb/library/aclib/aclib.h> #include <ydb/library/yql/utils/actor_log/log.h> - + #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/event_pb.h> #include <library/cpp/actors/core/hfunc.h> @@ -31,7 +31,7 @@ namespace NKqp { using namespace NKikimrConfig; using namespace NThreading; using namespace NYql; -using namespace NYql::NDq; +using namespace NYql::NDq; using namespace NRuCalc; static std::atomic<bool> FailForcedNewEngineExecution = false; @@ -569,7 +569,7 @@ public: } Y_VERIFY(QueryState); - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId, QueryState->TraceId); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId, QueryState->TraceId); if (ev->Get()->Finished) { QueryState->QueryResult = QueryState->AsyncQueryResult->GetResult(); @@ -684,7 +684,7 @@ public: return; } - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId); if (ev->Get()->Finished) { Y_VERIFY(CleanupState); @@ -886,7 +886,7 @@ private: void PerformQuery(const TActorContext& ctx) { Y_VERIFY(QueryState); auto requestInfo = TKqpRequestInfo(QueryState->TraceId, SessionId); - TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId, QueryState->TraceId); + TYqlLogScope logScope(ctx, NKikimrServices::KQP_YQL, SessionId, QueryState->TraceId); Gateway->SetToken(Settings.Cluster, QueryState->UserToken); auto& queryRequest = QueryState->Request; diff --git a/ydb/core/kqp/node/kqp_node.cpp b/ydb/core/kqp/node/kqp_node.cpp index 39594d2cbe..24f8f2ac79 100644 --- a/ydb/core/kqp/node/kqp_node.cpp +++ b/ydb/core/kqp/node/kqp_node.cpp @@ -233,7 +233,7 @@ private: memoryLimits.MkqlLightProgramMemoryLimit = Config.GetMkqlLightProgramMemoryLimit(); memoryLimits.MkqlHeavyProgramMemoryLimit = Config.GetMkqlHeavyProgramMemoryLimit(); if (Config.GetEnableInstantMkqlMemoryAlloc()) { - memoryLimits.AllocateMemoryFn = [rm = ResourceManager()](const auto& txId, ui64 taskId, ui64 memory) { + memoryLimits.AllocateMemoryFn = [rm = ResourceManager()](const auto& txId, ui64 taskId, ui64 memory) { NRm::TKqpResourcesRequest resources; resources.MemoryPool = NRm::EKqpMemoryPool::ScanQuery; resources.Memory = memory; diff --git a/ydb/core/kqp/opt/kqp_opt_effects.cpp b/ydb/core/kqp/opt/kqp_opt_effects.cpp index 653cb7b949..f5f2401ce9 100644 --- a/ydb/core/kqp/opt/kqp_opt_effects.cpp +++ b/ydb/core/kqp/opt/kqp_opt_effects.cpp @@ -69,7 +69,7 @@ TDqPhyPrecompute BuildPrecomputeStage(TExprBase expr, TExprContext& ctx) { .Build() .Build() .Build() - .Settings().Build() + .Settings().Build() .Done(); auto dqValue = Build<TDqCnValue>(ctx, expr.Pos()) @@ -248,7 +248,7 @@ bool BuildEffects(TPositionHandle pos, const TVector<TKqlTableEffect>& effects, .Add(newEffects) .Build() .Build() - .Settings().Build() + .Settings().Build() .Done(); for (ui32 i = 0; i < newEffects.size(); ++i) { diff --git a/ydb/core/kqp/prepare/kqp_query_plan.cpp b/ydb/core/kqp/prepare/kqp_query_plan.cpp index 00c616df6e..346a8b15cb 100644 --- a/ydb/core/kqp/prepare/kqp_query_plan.cpp +++ b/ydb/core/kqp/prepare/kqp_query_plan.cpp @@ -363,7 +363,7 @@ private: void WritePlanNodeToJson(const TQueryPlanNode& planNode, NJsonWriter::TBuf& writer) const { writer.BeginObject(); - + writer.WriteKey("PlanNodeId").WriteInt(planNode.NodeId); writer.WriteKey("Node Type").WriteString(planNode.TypeName); writer.WriteKey("StageGuid").WriteString(planNode.Guid); diff --git a/ydb/core/kqp/proxy/kqp_proxy_service.cpp b/ydb/core/kqp/proxy/kqp_proxy_service.cpp index f8f6efe61f..decd2cb301 100644 --- a/ydb/core/kqp/proxy/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy/kqp_proxy_service.cpp @@ -1242,7 +1242,7 @@ private: for (auto &entry : LogConfig.GetEntry()) { if (entry.GetComponent() == kqpYqlName && entry.HasLevel()) { auto yqlPriority = static_cast<NActors::NLog::EPriority>(entry.GetLevel()); - NYql::NDq::SetYqlLogLevels(yqlPriority); + NYql::NDq::SetYqlLogLevels(yqlPriority); KQP_PROXY_LOG_D("Updated YQL logs priority: " << (ui32)yqlPriority); return; } @@ -1253,7 +1253,7 @@ private: auto yqlPriority = static_cast<NActors::NLog::EPriority>(currentLevel); KQP_PROXY_LOG_D("Updated YQL logs priority to current level: " << (ui32)yqlPriority); - NYql::NDq::SetYqlLogLevels(yqlPriority); + NYql::NDq::SetYqlLogLevels(yqlPriority); } TKqpDbCountersPtr GetDbCountersForSession(const TString& sessionId) const { diff --git a/ydb/core/kqp/ut/kqp_explain_ut.cpp b/ydb/core/kqp/ut/kqp_explain_ut.cpp index aba927ba39..c3a0683636 100644 --- a/ydb/core/kqp/ut/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/kqp_explain_ut.cpp @@ -2,7 +2,7 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + namespace NKikimr { namespace NKqp { diff --git a/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp index 4d2babc7c4..16c89eceb5 100644 --- a/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp +++ b/ydb/core/kqp/ut/kqp_newengine_flowcontrol_ut.cpp @@ -5,7 +5,7 @@ #include <ydb/public/lib/experimental/ydb_experimental.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + namespace NKikimr::NKqp { using namespace NYdb; diff --git a/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp b/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp index 1e51940f5d..613724e640 100644 --- a/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_spilling_ut.cpp @@ -2,7 +2,7 @@ #include <ydb/core/kqp/counters/kqp_counters.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <util/system/fs.h> namespace NKikimr { diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index 5c72b3b20a..347ca35441 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -4,7 +4,7 @@ #include <ydb/public/lib/experimental/ydb_experimental.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <util/generic/size_literals.h> namespace NKikimr { diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp index 78a80f1d4a..f7815c016e 100644 --- a/ydb/core/kqp/ut/kqp_stats_ut.cpp +++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp @@ -3,7 +3,7 @@ #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <cstdlib> namespace NKikimr { diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index 49aacc946a..c17c8a7dc3 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -832,7 +832,7 @@ message TActivity { YQL_GET_CONNECTIONS_REQUEST_ACTOR = 527; YQL_MODIFY_CONNECTIONS_REQUEST_ACTOR = 528; YQL_MODIFY_HISTORY_REQUEST_ACTOR = 529; - YQL_WRITE_RESULT_DATA_ACTOR = 530; + YQL_WRITE_RESULT_DATA_ACTOR = 530; BS_SCRUB_ACTOR = 531; BS_BLOB_RECOVERY_ACTOR = 532; BS_RESTORE_CORRUPTED_BLOB_ACTOR = 533; diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h index 9adfd92cf9..298a8efbc0 100644 --- a/ydb/core/sys_view/common/scan_actor_base_impl.h +++ b/ydb/core/sys_view/common/scan_actor_base_impl.h @@ -10,7 +10,7 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/actors/core/hfunc.h> diff --git a/ydb/core/sys_view/nodes/nodes.cpp b/ydb/core/sys_view/nodes/nodes.cpp index fdc17542da..a2322de9f9 100644 --- a/ydb/core/sys_view/nodes/nodes.cpp +++ b/ydb/core/sys_view/nodes/nodes.cpp @@ -6,7 +6,7 @@ #include <ydb/core/node_whiteboard/node_whiteboard.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/core/hfunc.h> @@ -21,7 +21,7 @@ class TNodesScan : public TScanActorBase<TNodesScan> { public: using TBase = TScanActorBase<TNodesScan>; - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; } diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp index 003fead75a..d6ea9ff73b 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats.cpp +++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp @@ -6,7 +6,7 @@ #include <ydb/core/base/tablet_pipecache.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <library/cpp/actors/core/hfunc.h> namespace NKikimr { @@ -16,7 +16,7 @@ using namespace NActors; class TPartitionStatsCollector : public TActor<TPartitionStatsCollector> { public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::SYSTEM_VIEW_PART_STATS_COLLECTOR; } @@ -315,7 +315,7 @@ class TPartitionStatsScan : public TScanActorBase<TPartitionStatsScan> { public: using TBase = TScanActorBase<TPartitionStatsScan>; - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; } diff --git a/ydb/core/sys_view/query_stats/query_metrics.cpp b/ydb/core/sys_view/query_stats/query_metrics.cpp index 7c34e94cea..5f2acb5c23 100644 --- a/ydb/core/sys_view/query_stats/query_metrics.cpp +++ b/ydb/core/sys_view/query_stats/query_metrics.cpp @@ -29,7 +29,7 @@ class TQueryMetricsScan : public TScanActorBase<TQueryMetricsScan> { public: using TBase = TScanActorBase<TQueryMetricsScan>; - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; } diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp index 24d168352e..918c9b781e 100644 --- a/ydb/core/sys_view/query_stats/query_stats.cpp +++ b/ydb/core/sys_view/query_stats/query_stats.cpp @@ -10,7 +10,7 @@ #include <ydb/core/sys_view/service/sysview_service.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <library/cpp/actors/core/interconnect.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/core/hfunc.h> @@ -91,7 +91,7 @@ class TQueryStatsScan : public TScanActorBase<TQueryStatsScan<TGreater>> { public: using TBase = TScanActorBase<TQueryStatsScan<TGreater>>; - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; } diff --git a/ydb/core/sys_view/storage/groups.cpp b/ydb/core/sys_view/storage/groups.cpp index 92ee638444..dabd46f9dc 100644 --- a/ydb/core/sys_view/storage/groups.cpp +++ b/ydb/core/sys_view/storage/groups.cpp @@ -4,7 +4,7 @@ namespace NKikimr::NSysView { template<> void SetField<0>(NKikimrSysView::TGroupKey& key, ui32 value) { key.SetGroupId(value); } - + class TGroupsScan : public TStorageScanBase<TGroupsScan, TEvSysView::TEvGetGroupsResponse> { public: using TStorageScanBase::TStorageScanBase; diff --git a/ydb/core/sys_view/storage/pdisks.cpp b/ydb/core/sys_view/storage/pdisks.cpp index d091a7ce53..47f84ae9aa 100644 --- a/ydb/core/sys_view/storage/pdisks.cpp +++ b/ydb/core/sys_view/storage/pdisks.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NSysView { template<> void SetField<0>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetNodeId(value); } template<> void SetField<1>(NKikimrSysView::TPDiskKey& key, ui32 value) { key.SetPDiskId(value); } - + class TPDisksScan : public TStorageScanBase<TPDisksScan, TEvSysView::TEvGetPDisksResponse> { public: using TStorageScanBase::TStorageScanBase; diff --git a/ydb/core/sys_view/storage/storage_pools.cpp b/ydb/core/sys_view/storage/storage_pools.cpp index bd54340e24..66896a6999 100644 --- a/ydb/core/sys_view/storage/storage_pools.cpp +++ b/ydb/core/sys_view/storage/storage_pools.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NSysView { template<> void SetField<0>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetBoxId(value); } template<> void SetField<1>(NKikimrSysView::TStoragePoolKey& key, ui64 value) { key.SetStoragePoolId(value); } - + class TStoragePoolsScan : public TStorageScanBase<TStoragePoolsScan, TEvSysView::TEvGetStoragePoolsResponse> { public: using TStorageScanBase::TStorageScanBase; diff --git a/ydb/core/sys_view/storage/vslots.cpp b/ydb/core/sys_view/storage/vslots.cpp index d69a2c33c9..987eb2750d 100644 --- a/ydb/core/sys_view/storage/vslots.cpp +++ b/ydb/core/sys_view/storage/vslots.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NSysView { template<> void SetField<0>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetNodeId(value); } template<> void SetField<1>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetPDiskId(value); } template<> void SetField<2>(NKikimrSysView::TVSlotKey& key, ui32 value) { key.SetVSlotId(value); } - + class TVSlotsScan : public TStorageScanBase<TVSlotsScan, TEvSysView::TEvGetVSlotsResponse> { public: using TStorageScanBase::TStorageScanBase; diff --git a/ydb/core/sys_view/tablets/tablets.cpp b/ydb/core/sys_view/tablets/tablets.cpp index 638f2a42a3..88627748d8 100644 --- a/ydb/core/sys_view/tablets/tablets.cpp +++ b/ydb/core/sys_view/tablets/tablets.cpp @@ -8,7 +8,7 @@ #include <ydb/core/mind/hive/tablet_info.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <library/cpp/actors/core/hfunc.h> namespace NKikimr::NSysView { @@ -19,7 +19,7 @@ class TTabletsScan : public TScanActorBase<TTabletsScan> { public: using TBase = TScanActorBase<TTabletsScan>; - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_SYSTEM_VIEW_SCAN; } diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index e4ed25d4f0..63f0436bff 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -171,7 +171,7 @@ namespace NActors { } void TTestActorRuntime::InitActorSystemSetup(TActorSystemSetup& setup) { - setup.MaxActivityType = GetActivityTypeCount(); + setup.MaxActivityType = GetActivityTypeCount(); } NKikimr::TAppData& TTestActorRuntime::GetAppData(ui32 nodeIndex) { diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index df027e0c71..d4907c26f1 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -124,8 +124,8 @@ namespace Tests { NMiniKQL::IFunctionRegistry* DefaultFrFactory(const NScheme::TTypeRegistry& typeRegistry) { Y_UNUSED(typeRegistry); // register test UDFs - auto freg = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); - NKikimr::NMiniKQL::FillStaticModules(*freg); + auto freg = NKikimr::NMiniKQL::CreateFunctionRegistry(NKikimr::NMiniKQL::CreateBuiltinRegistry())->Clone(); + NKikimr::NMiniKQL::FillStaticModules(*freg); return freg.Release(); } @@ -195,10 +195,10 @@ namespace Tests { app.SetFnRegistry(Settings->FrFactory); app.SetFormatsFactory(Settings->Formats); - if (Settings->Formats) { - NKikHouse::RegisterFormat(*Settings->Formats); - } - + if (Settings->Formats) { + NKikHouse::RegisterFormat(*Settings->Formats); + } + NKikimr::SetupChannelProfiles(app, Settings->Domain); Runtime->SetupMonitoring(); diff --git a/ydb/core/tx/columnshard/blob_cache.cpp b/ydb/core/tx/columnshard/blob_cache.cpp index 2037394b91..1d676cd21a 100644 --- a/ydb/core/tx/columnshard/blob_cache.cpp +++ b/ydb/core/tx/columnshard/blob_cache.cpp @@ -66,7 +66,7 @@ private: const TCounterPtr ReadsInQueue; public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::BLOB_CACHE_ACTOR; } diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index c01a3282e6..e2f7ba5a51 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -24,7 +24,7 @@ constexpr TDuration SCAN_HARD_TIMEOUT_GAP = TDuration::Seconds(5); class TColumnShardScan : public TActorBootstrapped<TColumnShardScan>, NArrow::IRowWriter { public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_OLAP_SCAN; } diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp index ba24d262fd..b6b6b7e337 100644 --- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp @@ -29,7 +29,7 @@ class TKqpScanResult : public IDestructable {}; class TKqpScan : public TActor<TKqpScan>, public NTable::IScan { public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::KQP_TABLE_SCAN; } diff --git a/ydb/core/tx/datashard/datashard__stats.cpp b/ydb/core/tx/datashard/datashard__stats.cpp index 7edb305e6c..129a14dca5 100644 --- a/ydb/core/tx/datashard/datashard__stats.cpp +++ b/ydb/core/tx/datashard/datashard__stats.cpp @@ -23,7 +23,7 @@ public: , SearchHeight(searchHeight) {} - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return NKikimrServices::TActivity::DATASHARD_STATS_BUILDER; } diff --git a/ydb/core/tx/datashard/datashard_ut_kqp.cpp b/ydb/core/tx/datashard/datashard_ut_kqp.cpp index dc6d374224..d3145951a4 100644 --- a/ydb/core/tx/datashard/datashard_ut_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_ut_kqp.cpp @@ -8,7 +8,7 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + using namespace NKikimr; using namespace NKikimr::NDataShard; using namespace NKikimr::NDataShard::NKqpHelpers; diff --git a/ydb/core/tx/scheme_board/monitoring.cpp b/ydb/core/tx/scheme_board/monitoring.cpp index d9382fa447..deda604655 100644 --- a/ydb/core/tx/scheme_board/monitoring.cpp +++ b/ydb/core/tx/scheme_board/monitoring.cpp @@ -1096,7 +1096,7 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { } public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return TActivity::SCHEME_BOARD_INFO_REQUESTER_ACTOR; } @@ -1312,7 +1312,7 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { } public: - static constexpr auto ActorActivityType() { + static constexpr auto ActorActivityType() { return TActivity::SCHEME_BOARD_MONITORING_ACTOR; } diff --git a/ydb/core/yq/libs/actors/database_resolver.cpp b/ydb/core/yq/libs/actors/database_resolver.cpp index 02d4b0e419..c784d80c00 100644 --- a/ydb/core/yq/libs/actors/database_resolver.cpp +++ b/ydb/core/yq/libs/actors/database_resolver.cpp @@ -1,58 +1,58 @@ -#include "database_resolver.h" - +#include "database_resolver.h" + #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/common/cache.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/http/http.h> -#include <library/cpp/actors/http/http_proxy.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/http/http.h> +#include <library/cpp/actors/http/http_proxy.h> +#include <library/cpp/json/json_reader.h> #include <ydb/core/protos/services.pb.h> - + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "DatabaseResolver - TraceId: " << TraceId << ": " << stream) - + #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "DatabaseResolver - TraceId: " << TraceId << ": " << stream) - + namespace NYq { - -using namespace NActors; + +using namespace NActors; using namespace NYql; -using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; - -using TParsers = THashMap<DatabaseType, std::function<TEndpoint(NJson::TJsonValue& body, bool)>>; +using TEndpoint = TEvents::TEvEndpointResponse::TEndpoint; + +using TParsers = THashMap<DatabaseType, std::function<TEndpoint(NJson::TJsonValue& body, bool)>>; using TCache = TTtlCache<std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>, std::variant<TEndpoint, TString>>; - + TString TransformMdbHostToCorrectFormat(const TString& mdbHost) { return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443"; } class TResponseProcessor : public TActorBootstrapped<TResponseProcessor> -{ -public: +{ +public: enum EWakeUp { WU_Die_On_Ttl }; - TResponseProcessor( + TResponseProcessor( const TActorId sender, - TCache& cache, + TCache& cache, const THashMap<std::pair<TString, DatabaseType>, TEndpoint>& ready, const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>>& requests, const TString& traceId, - bool mdbTransformHost, - const TParsers& parsers) + bool mdbTransformHost, + const TParsers& parsers) : Sender(sender) - , Cache(cache) - , Requests(requests) - , TraceId(traceId) + , Cache(cache) + , Requests(requests) + , TraceId(traceId) , MdbTransformHost(mdbTransformHost) - , DatabaseId2Endpoint(ready) - , Parsers(parsers) - { } - + , DatabaseId2Endpoint(ready) + , Parsers(parsers) + { } + static constexpr char ActorName[] = "YQ_RESPONSE_PROCESSOR"; void Bootstrap() { @@ -64,8 +64,8 @@ public: hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle); hFunc(NActors::TEvents::TEvWakeup, HandleWakeup) cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); - }); - + }); + private: void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev) { auto tag = ev->Get()->Tag; @@ -102,59 +102,59 @@ private: } void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) - { - TString status; - TString errorMessage; - TString databaseId; + { + TString status; + TString errorMessage; + TString databaseId; DatabaseType databaseType = DatabaseType::Ydb; TEvents::TDatabaseAuth info; - TMaybe<TEndpoint> result; + TMaybe<TEndpoint> result; HandledIds++; - if (ev->Get()->Error.empty() && (ev->Get()->Response && ((status = ev->Get()->Response->Status) == "200"))) { - NJson::TJsonReaderConfig jsonConfig; - NJson::TJsonValue databaseInfo; - auto it = Requests.find(ev->Get()->Request); - + if (ev->Get()->Error.empty() && (ev->Get()->Response && ((status = ev->Get()->Response->Status) == "200"))) { + NJson::TJsonReaderConfig jsonConfig; + NJson::TJsonValue databaseInfo; + auto it = Requests.find(ev->Get()->Request); + LOG_D("Got databaseId response " << ev->Get()->Response->Body); - if (it == Requests.end()) { - errorMessage = "Unknown databaseId"; - } else { + if (it == Requests.end()) { + errorMessage = "Unknown databaseId"; + } else { std::tie(databaseId, databaseType, info) = it->second; const bool parseJsonOk = NJson::ReadJsonTree(ev->Get()->Response->Body, &jsonConfig, &databaseInfo); TParsers::const_iterator parserIt; if (parseJsonOk && (parserIt = Parsers.find(databaseType)) != Parsers.end()) { - try { + try { auto res = parserIt->second(databaseInfo, MdbTransformHost); LOG_D("Got " << databaseId << " " << databaseType << " endpoint " << res.Endpoint << " " << res.Database); DatabaseId2Endpoint[std::make_pair(databaseId, databaseType)] = res; - result.ConstructInPlace(res); - } catch (...) { - errorMessage = TStringBuilder() - << " Couldn't resolve " + result.ConstructInPlace(res); + } catch (...) { + errorMessage = TStringBuilder() + << " Couldn't resolve " << databaseType << " Id: " - << databaseId << "\n" - << CurrentExceptionMessage(); + << databaseId << "\n" + << CurrentExceptionMessage(); } - } else { + } else { errorMessage = TStringBuilder() << "Unable to parse database information." << "Database Id: " << databaseId << ", Database Type: " << databaseType; - } - } - } else { - errorMessage = ev->Get()->Error; + } + } + } else { + errorMessage = ev->Get()->Error; const TString error = "Cannot resolve databaseId (status = " + ToString(status) + ")"; if (!errorMessage.empty()) { errorMessage += '\n'; - } + } errorMessage += error; - } - - if (errorMessage) { + } + + if (errorMessage) { LOG_E("Error on response parsing: " << errorMessage); Success = false; - } - + } + if (databaseId) { auto key = std::make_tuple(databaseId, databaseType, info); if (errorMessage) { @@ -163,102 +163,102 @@ private: Cache.Put(key, result); } } - + if (HandledIds == Requests.size()) { SendResolvedEndpointsAndDie(); - } - + } + LOG_D(DatabaseId2Endpoint.size() << " of " << Requests.size() << " done"); - } - + } + private: - const TActorId Sender; - TCache& Cache; + const TActorId Sender; + TCache& Cache; const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> Requests; - const TString TraceId; + const TString TraceId; const bool MdbTransformHost; THashMap<std::pair<TString, DatabaseType>, TEvents::TEvEndpointResponse::TEndpoint> DatabaseId2Endpoint; size_t HandledIds = 0; bool Success = true; - const TParsers& Parsers; + const TParsers& Parsers; TDuration ResolvingTtl = TDuration::Seconds(30); //TODO: Use cfg -}; - -class TDatabaseResolver: public TActor<TDatabaseResolver> -{ -public: +}; + +class TDatabaseResolver: public TActor<TDatabaseResolver> +{ +public: TDatabaseResolver(TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) - : TActor<TDatabaseResolver>(&TDatabaseResolver::State) - , HttpProxy(httpProxy) + : TActor<TDatabaseResolver>(&TDatabaseResolver::State) + , HttpProxy(httpProxy) , CredentialsFactory(credentialsFactory) - , Cache( - TTtlCacheSettings() - .SetTtl(TDuration::Minutes(10)) - .SetErrorTtl(TDuration::Minutes(1)) - .SetMaxSize(1000000)) - { - auto ydbParser = [](NJson::TJsonValue& databaseInfo, bool) { - bool secure = false; + , Cache( + TTtlCacheSettings() + .SetTtl(TDuration::Minutes(10)) + .SetErrorTtl(TDuration::Minutes(1)) + .SetMaxSize(1000000)) + { + auto ydbParser = [](NJson::TJsonValue& databaseInfo, bool) { + bool secure = false; TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); - TString prefix("/?database="); - TString database; - auto pos = endpoint.rfind(prefix); - if (pos != TString::npos) { - database = endpoint.substr(pos+prefix.length()); - endpoint = endpoint.substr(0, pos); - } - if (endpoint.StartsWith("grpcs://")) { - secure = true; - } - pos = endpoint.find("://"); - if (pos != TString::npos) { - endpoint = endpoint.substr(pos+3); - } - + TString prefix("/?database="); + TString database; + auto pos = endpoint.rfind(prefix); + if (pos != TString::npos) { + database = endpoint.substr(pos+prefix.length()); + endpoint = endpoint.substr(0, pos); + } + if (endpoint.StartsWith("grpcs://")) { + secure = true; + } + pos = endpoint.find("://"); + if (pos != TString::npos) { + endpoint = endpoint.substr(pos+3); + } + Y_ENSURE(endpoint); - return TEvents::TEvEndpointResponse::TEndpoint{endpoint, database, secure}; - }; - Parsers[DatabaseType::Ydb] = ydbParser; - Parsers[DatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, bool mdbTransformHost) - { - auto ret = ydbParser(databaseInfo, mdbTransformHost); - // TODO: Take explicit field from MVP - if (ret.Endpoint.StartsWith("ydb.")) { - // Replace "ydb." -> "yds." - ret.Endpoint[2] = 's'; - } - return ret; - }; - Parsers[DatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, bool mdbTransformHost) { - TString endpoint; - TVector<TString> aliveHosts; + return TEvents::TEvEndpointResponse::TEndpoint{endpoint, database, secure}; + }; + Parsers[DatabaseType::Ydb] = ydbParser; + Parsers[DatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, bool mdbTransformHost) + { + auto ret = ydbParser(databaseInfo, mdbTransformHost); + // TODO: Take explicit field from MVP + if (ret.Endpoint.StartsWith("ydb.")) { + // Replace "ydb." -> "yds." + ret.Endpoint[2] = 's'; + } + return ret; + }; + Parsers[DatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, bool mdbTransformHost) { + TString endpoint; + TVector<TString> aliveHosts; for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { - if (host["health"].GetString() == "ALIVE" && host["type"].GetString() == "CLICKHOUSE") { - aliveHosts.push_back(host["name"].GetString()); - } - } - if (!aliveHosts.empty()) { - endpoint = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())]; - } - if (!endpoint) { - ythrow yexception() << "No ALIVE ClickHouse hosts exist"; - } - endpoint = mdbTransformHost ? TransformMdbHostToCorrectFormat(endpoint) : endpoint; - return TEvents::TEvEndpointResponse::TEndpoint{endpoint, "", true}; - }; - } - + if (host["health"].GetString() == "ALIVE" && host["type"].GetString() == "CLICKHOUSE") { + aliveHosts.push_back(host["name"].GetString()); + } + } + if (!aliveHosts.empty()) { + endpoint = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())]; + } + if (!endpoint) { + ythrow yexception() << "No ALIVE ClickHouse hosts exist"; + } + endpoint = mdbTransformHost ? TransformMdbHostToCorrectFormat(endpoint) : endpoint; + return TEvents::TEvEndpointResponse::TEndpoint{endpoint, "", true}; + }; + } + static constexpr char ActorName[] = "YQ_DATABASE_RESOLVER"; -private: - STRICT_STFUNC(State, { - HFunc(TEvents::TEvEndpointRequest, Handle); - cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); - }); - - void Handle(TEvents::TEvEndpointRequest::TPtr ev, const TActorContext& ctx) - { - TraceId = ev->Get()->TraceId; +private: + STRICT_STFUNC(State, { + HFunc(TEvents::TEvEndpointRequest, Handle); + cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); + }); + + void Handle(TEvents::TEvEndpointRequest::TPtr ev, const TActorContext& ctx) + { + TraceId = ev->Get()->TraceId; LOG_D("Start databaseId resolver for " << ev->Get()->DatabaseIds.size() << " ids"); THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, DatabaseType, TEvents::TDatabaseAuth>> requests; // request, (dbId, type, info) THashMap<std::pair<TString, DatabaseType>, TEndpoint> ready; @@ -266,21 +266,21 @@ private: const auto& [databaseId, type] = p; TMaybe<std::variant<TEndpoint, TString>> endpoint; auto key = std::make_tuple(databaseId, type, info); - if (Cache.Get(key, &endpoint)) { - if (endpoint) { + if (Cache.Get(key, &endpoint)) { + if (endpoint) { ready.insert(std::make_pair(p, (endpoint->index() == 0 ? std::get<0>(*endpoint) : TEndpoint{}) )); - } - continue; - } - + } + continue; + } + try { TString url = IsIn({ DatabaseType::Ydb, DatabaseType::DataStreams }, type) ? ev->Get()->YdbMvpEndpoint + "/database?databaseId=" + databaseId : ev->Get()->MdbGateway + "/managed-clickhouse/v1/clusters/" + databaseId + "/hosts"; LOG_D("Get '" << url << "'"); - + NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestGet(url); auto credentialsProviderFactory = CreateCredentialsProviderFactoryForStructuredToken(CredentialsFactory, info.StructuredToken, info.AddBearerToToken); @@ -294,31 +294,31 @@ private: const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId " << databaseId << ", details: " << e.what(); LOG_E(msg); Cache.Put(key, endpoint); - } - } - - if (!requests.empty()) { + } + } + + if (!requests.empty()) { auto helper = Register( new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbTransformHost, Parsers)); - - for (const auto& [request, _] : requests) { - ctx.Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request))); - } - } else { + + for (const auto& [request, _] : requests) { + ctx.Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request))); + } + } else { Send(ev->Sender, new TEvents::TEvEndpointResponse(std::move(ready), /*success=*/true)); - } - } - + } + } + private: - TParsers Parsers; - const TActorId HttpProxy; + TParsers Parsers; + const TActorId HttpProxy; const ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; - TString TraceId; - TCache Cache; -}; - + TString TraceId; + TCache Cache; +}; + NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { return new TDatabaseResolver(httpProxy, credentialsFactory); -} - +} + } /* namespace NYq */ diff --git a/ydb/core/yq/libs/actors/database_resolver.h b/ydb/core/yq/libs/actors/database_resolver.h index 148631e20a..3e50f46bc4 100644 --- a/ydb/core/yq/libs/actors/database_resolver.h +++ b/ydb/core/yq/libs/actors/database_resolver.h @@ -1,12 +1,12 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> +#pragma once + +#include <library/cpp/actors/core/actor.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> - + namespace NYq { - + TString TransformMdbHostToCorrectFormat(const TString& mdbHost); NActors::IActor* CreateDatabaseResolver(NActors::TActorId httpProxy, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory); - + } /* namespace NYq */ diff --git a/ydb/core/yq/libs/actors/error.cpp b/ydb/core/yq/libs/actors/error.cpp index 1886528d86..7a847961c1 100644 --- a/ydb/core/yq/libs/actors/error.cpp +++ b/ydb/core/yq/libs/actors/error.cpp @@ -1,11 +1,11 @@ -#include "proxy.h" - -#include <util/system/hostname.h> - +#include "proxy.h" + +#include <util/system/hostname.h> + namespace NYq { - -TString MakeInternalError(const TString& text) { - return TStringBuilder() << "Internal error (" << text << ", host: " << HostName() << ")"; -} - + +TString MakeInternalError(const TString& text) { + return TStringBuilder() << "Internal error (" << text << ", host: " << HostName() << ")"; +} + } // namespace NYq diff --git a/ydb/core/yq/libs/actors/pending_fetcher.cpp b/ydb/core/yq/libs/actors/pending_fetcher.cpp index 13e9064d26..0095963c8f 100644 --- a/ydb/core/yq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/yq/libs/actors/pending_fetcher.cpp @@ -1,16 +1,16 @@ #include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> -#include "proxy.h" +#include "proxy.h" #include "nodes_manager.h" - -#include "database_resolver.h" - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> + +#include "database_resolver.h" + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> #include <library/cpp/protobuf/interop/cast.h> #include <ydb/core/protos/services.pb.h> - + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/utils/actor_log/log.h> #include <ydb/library/yql/core/services/mounts/yql_mounts.h> @@ -26,7 +26,7 @@ #include <ydb/library/yql/providers/ydb/provider/yql_ydb_provider.h> #include <ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_provider.h> #include <ydb/library/yql/sql/settings/translation_settings.h> -#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/yson/node/node_io.h> #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_program_builder.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> @@ -36,38 +36,38 @@ #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/protos/issue_message.pb.h> - + #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> - + #include <ydb/core/yq/libs/common/entity_id.h> #include <ydb/core/yq/libs/events/events.h> - + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/private_client/private_client.h> -#include <library/cpp/actors/core/log.h> - +#include <library/cpp/actors/core/log.h> + #include <ydb/library/security/util.h> -#include <util/generic/deque.h> -#include <util/generic/guid.h> -#include <util/system/hostname.h> - +#include <util/generic/deque.h> +#include <util/generic/guid.h> +#include <util/system/hostname.h> + #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Fetcher: " << stream) #define LOG_I(stream) \ LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Fetcher: " << stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Fetcher: " << stream) - + namespace NYq { - -using namespace NActors; - + +using namespace NActors; + namespace { struct TEvGetTaskInternalResponse : public NActors::TEventLocal<TEvGetTaskInternalResponse, NActors::TEvents::TSystem::Completed> { @@ -92,35 +92,35 @@ TVector<TElement> VectorFromProto(const ::google::protobuf::RepeatedPtrField<TEl } // namespace -class TYqlPendingFetcher : public NActors::TActorBootstrapped<TYqlPendingFetcher> { -public: - TYqlPendingFetcher( +class TYqlPendingFetcher : public NActors::TActorBootstrapped<TYqlPendingFetcher> { +public: + TYqlPendingFetcher( const NYq::TYqSharedResources::TPtr& yqSharedResources, const ::NYq::NConfig::TCommonConfig& commonConfig, const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig, const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig, const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, const ::NYq::NCommon::TServiceCounters& serviceCounters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr s3Gateway, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const NMonitoring::TDynamicCounterPtr& clientCounters - ) + ) : YqSharedResources(yqSharedResources) , CommonConfig(commonConfig) , CheckpointCoordinatorConfig(checkpointCoordinatorConfig) , PrivateApiConfig(privateApiConfig) , GatewaysConfig(gatewaysConfig) , PingerConfig(pingerConfig) - , FunctionRegistry(functionRegistry) - , TimeProvider(timeProvider) - , RandomProvider(randomProvider) - , DqCompFactory(dqCompFactory) + , FunctionRegistry(functionRegistry) + , TimeProvider(timeProvider) + , RandomProvider(randomProvider) + , DqCompFactory(dqCompFactory) , ServiceCounters(serviceCounters, "pending_fetcher") , CredentialsFactory(credentialsFactory) , S3Gateway(s3Gateway) @@ -133,48 +133,48 @@ public: .DiscoveryEndpoint(PrivateApiConfig.GetTaskServiceEndpoint()) .Database(PrivateApiConfig.GetTaskServiceDatabase() ? PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), ClientCounters) - { + { Y_ENSURE(GetYqlDefaultModuleResolverWithContext(ModuleResolver)); - } - + } + static constexpr char ActorName[] = "YQ_PENDING_FETCHER"; - - void PassAway() final { + + void PassAway() final { LOG_D("Stop Fetcher"); - Send(DatabaseResolver, new NActors::TEvents::TEvPoison()); - NActors::IActor::PassAway(); - } - - void Bootstrap(const TActorContext& ctx) { - Become(&TYqlPendingFetcher::StateFunc); - - Y_UNUSED(ctx); - + Send(DatabaseResolver, new NActors::TEvents::TEvPoison()); + NActors::IActor::PassAway(); + } + + void Bootstrap(const TActorContext& ctx) { + Become(&TYqlPendingFetcher::StateFunc); + + Y_UNUSED(ctx); + DatabaseResolver = Register(CreateDatabaseResolver(MakeYqlAnalyticsHttpProxyId(), CredentialsFactory)); Send(SelfId(), new NActors::TEvents::TEvWakeup()); - + LOG_I("STARTED"); LogScope.ConstructInPlace(NActors::TActivationContext::ActorSystem(), NKikimrServices::YQL_PROXY, Guid); - } - -private: - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) { - LOG_E("TYqlPendingFetcher::OnUndelivered"); - - HasRunningRequest = false; - } - - void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&, const NActors::TActorContext&) { - Schedule(PendingFetchPeriod, new NActors::TEvents::TEvWakeup()); - - if (!HasRunningRequest) { - HasRunningRequest = true; - GetPendingTask(); - } - } - + } + +private: + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext&) { + LOG_E("TYqlPendingFetcher::OnUndelivered"); + + HasRunningRequest = false; + } + + void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&, const NActors::TActorContext&) { + Schedule(PendingFetchPeriod, new NActors::TEvents::TEvWakeup()); + + if (!HasRunningRequest) { + HasRunningRequest = true; + GetPendingTask(); + } + } + void HandleResponse(TEvGetTaskInternalResponse::TPtr& ev) { - HasRunningRequest = false; + HasRunningRequest = false; LOG_D("Got GetTask response from PrivateApi"); if (!ev->Get()->Success) { LOG_E("Error with GetTask: "<< ev->Get()->Issues.ToString()); @@ -190,7 +190,7 @@ private: GetPendingTask(); } } - + void GetPendingTask() { LOG_D("Request Private::GetTask" << ", Owner: " << Guid << ", Host: " << HostName()); Yq::Private::GetTaskRequest request; @@ -233,7 +233,7 @@ private: serviceAccounts[identity.value()] = identity.signature(); } - TRunActorParams params( + TRunActorParams params( YqSharedResources->YdbDriver, S3Gateway, FunctionRegistry, RandomProvider, ModuleResolver, ModuleResolver->GetNextUniqueId(), @@ -265,37 +265,37 @@ private: NDq::SetYqlLogLevels(NActors::NLog::PRI_TRACE); Register(CreateRunActor(ServiceCounters, std::move(params))); - } - - STRICT_STFUNC( - StateFunc, - - HFunc(NActors::TEvents::TEvWakeup, HandleWakeup) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + } + + STRICT_STFUNC( + StateFunc, + + HFunc(NActors::TEvents::TEvWakeup, HandleWakeup) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) hFunc(TEvGetTaskInternalResponse, HandleResponse) - ); - + ); + NYq::TYqSharedResources::TPtr YqSharedResources; NYq::NConfig::TCommonConfig CommonConfig; NYq::NConfig::TCheckpointCoordinatorConfig CheckpointCoordinatorConfig; NYq::NConfig::TPrivateApiConfig PrivateApiConfig; NYq::NConfig::TGatewaysConfig GatewaysConfig; NYq::NConfig::TPingerConfig PingerConfig; - - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; - TIntrusivePtr<ITimeProvider> TimeProvider; - TIntrusivePtr<IRandomProvider> RandomProvider; - NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory; - TIntrusivePtr<IDqGateway> DqGateway; + + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + TIntrusivePtr<ITimeProvider> TimeProvider; + TIntrusivePtr<IRandomProvider> RandomProvider; + NKikimr::NMiniKQL::TComputationNodeFactory DqCompFactory; + TIntrusivePtr<IDqGateway> DqGateway; ::NYq::NCommon::TServiceCounters ServiceCounters; - - IModuleResolver::TPtr ModuleResolver; - - bool HasRunningRequest = false; - const TDuration PendingFetchPeriod = TDuration::Seconds(1); - - TActorId DatabaseResolver; - + + IModuleResolver::TPtr ModuleResolver; + + bool HasRunningRequest = false; + const TDuration PendingFetchPeriod = TDuration::Seconds(1); + + TActorId DatabaseResolver; + ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; const IHTTPGateway::TPtr S3Gateway; const ::NPq::NConfigurationManager::IConnections::TPtr PqCmConnections; @@ -305,47 +305,47 @@ private: TPrivateClient Client; TMaybe<NYql::NLog::TScopedBackend<NYql::NDq::TYqlLogScope>> LogScope; -}; - - -NActors::IActor* CreatePendingFetcher( +}; + + +NActors::IActor* CreatePendingFetcher( const NYq::TYqSharedResources::TPtr& yqSharedResources, const ::NYq::NConfig::TCommonConfig& commonConfig, const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig, const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig, const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, const ::NYq::NCommon::TServiceCounters& serviceCounters, ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, IHTTPGateway::TPtr s3Gateway, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const NMonitoring::TDynamicCounterPtr& clientCounters) -{ - return new TYqlPendingFetcher( +{ + return new TYqlPendingFetcher( yqSharedResources, commonConfig, checkpointCoordinatorConfig, privateApiConfig, gatewaysConfig, pingerConfig, - functionRegistry, - timeProvider, - randomProvider, - dqCompFactory, + functionRegistry, + timeProvider, + randomProvider, + dqCompFactory, serviceCounters, credentialsFactory, s3Gateway, std::move(pqCmConnections), clientCounters); -} - -TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) { - constexpr TStringBuf name = "YQLFETCHER"; - return NActors::TActorId(nodeId, name); -} - +} + +TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId) { + constexpr TStringBuf name = "YQLFETCHER"; + return NActors::TActorId(nodeId, name); +} + } /* NYq */ diff --git a/ydb/core/yq/libs/actors/pinger.cpp b/ydb/core/yq/libs/actors/pinger.cpp index 128b29fbd6..55fc80d6a0 100644 --- a/ydb/core/yq/libs/actors/pinger.cpp +++ b/ydb/core/yq/libs/actors/pinger.cpp @@ -1,48 +1,48 @@ #include <ydb/core/yq/libs/config/protos/pinger.pb.h> -#include "proxy.h" +#include "proxy.h" #include <util/datetime/base.h> - + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/events/events.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/log.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> - + #include <util/generic/utility.h> #include <deque> #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream) - + #define LOG_W(stream) \ LOG_WARN_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream) - + #define LOG_T(stream) \ LOG_TRACE_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Pinger - " << "QueryId: " << Id << ", Owner: " << OwnerId << " " << stream) namespace NYq { - -using namespace NActors; + +using namespace NActors; using namespace NYql; - + struct TEvPingResponse : public NActors::TEventLocal<TEvPingResponse, NActors::TEvents::TSystem::CallbackCompletion> { TPingTaskResult Result; YandexQuery::QueryAction Action = YandexQuery::QUERY_ACTION_UNSPECIFIED; - + explicit TEvPingResponse(TPingTaskResult&& result) : Result(std::move(result)) , Action(Result.IsResultSet() ? Result.GetResult().action() : YandexQuery::QUERY_ACTION_UNSPECIFIED) - { - } - + { + } + explicit TEvPingResponse(const TString& errorMessage) : TEvPingResponse(MakeResultFromErrorMessage(errorMessage)) { @@ -54,9 +54,9 @@ private: issues.AddIssue(errorMessage); return TPingTaskResult(NYdb::TStatus(NYdb::EStatus::INTERNAL_ERROR, std::move(issues)), nullptr); } -}; - -class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { +}; + +class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { class TRetryState { public: void Init(const TInstant& now, const TInstant& startLeaseTime, const TDuration& maxRetryTime) { @@ -140,50 +140,50 @@ class TPingerActor : public NActors::TActorBootstrapped<TPingerActor> { } }; -public: - TPingerActor( - const TScope& scope, - const TString& userId, - const TString& id, - const TString& ownerId, +public: + TPingerActor( + const TScope& scope, + const TString& userId, + const TString& id, + const TString& ownerId, const TPrivateClient& client, const TActorId parent, const NConfig::TPingerConfig& config, const TInstant& deadline) : Config(config) , Scope(scope) - , UserId(userId) - , Id(id) - , OwnerId(ownerId) - , Client(client) + , UserId(userId) + , Id(id) + , OwnerId(ownerId) + , Client(client) , Parent(parent) , Deadline(deadline) { } - + static constexpr char ActorName[] = "YQ_PINGER"; - + void Bootstrap() { - LOG_D("Start Pinger"); + LOG_D("Start Pinger"); StartLeaseTime = TActivationContext::Now(); // Not accurate value, but it allows us to retry the first unsuccessful ping request. ScheduleNextPing(); - Become(&TPingerActor::StateFunc); - } - -private: - STRICT_STFUNC( - StateFunc, + Become(&TPingerActor::StateFunc); + } + +private: + STRICT_STFUNC( + StateFunc, cFunc(NActors::TEvents::TEvPoison::EventType, PassAway) hFunc(NActors::TEvents::TEvWakeup, Wakeup) hFunc(TEvPingResponse, Handle) hFunc(TEvents::TEvForwardPingRequest, Handle) ) - - void PassAway() override { - LOG_D("Stop Pinger"); + + void PassAway() override { + LOG_D("Stop Pinger"); NActors::TActorBootstrapped<TPingerActor>::PassAway(); - } - + } + void ScheduleNextPing() { if (!Finishing) { SchedulerCookieHolder.Reset(ISchedulerCookie::Make2Way()); @@ -213,12 +213,12 @@ private: } void WakeupContinueLease() { - SchedulerCookieHolder.Reset(nullptr); + SchedulerCookieHolder.Reset(nullptr); if (!Finishing) { Ping(); } } - + void WakeupRetryContinueLease() { Ping(true); } @@ -248,14 +248,14 @@ private: ForwardRequests.emplace_back(std::move(ev)); ForwardPing(); } - } - + } + void SendQueryAction(YandexQuery::QueryAction action) { if (!Finishing) { Send(Parent, new TEvents::TEvQueryActionResult(action)); - } - } - + } + } + static bool Retryable(TEvPingResponse::TPtr& ev) { if (ev->Get()->Result.IsTransportError()) { return true; @@ -342,14 +342,14 @@ private: Send(Parent, new TEvents::TEvForwardPingResponse(false, ev->Get()->Action), 0, ev->Cookie); FatalError = true; ForwardRequests.clear(); - } - + } + if (Finishing && ForwardRequests.empty() && !Requested) { LOG_D("Query finished"); PassAway(); - } - } - + } + } + void ForwardPing(bool retry = false) { Y_VERIFY(!ForwardRequests.empty()); auto& reqInfo = ForwardRequests.front(); @@ -360,7 +360,7 @@ private: reqInfo.RetryState.Init(TActivationContext::Now(), StartLeaseTime, Config.PingPeriod); } LOG_D((retry ? "Retry forward" : "Forward") << " request Private::PingTask"); - + Ping(reqInfo.Request->Get()->Request, reqInfo.Request->Cookie); } } @@ -391,19 +391,19 @@ private: future.Subscribe( [actorSystem, selfId, cookie, future](const NThreading::TFuture<TPingTaskResult>&) mutable { std::unique_ptr<TEvPingResponse> ev; - try { + try { auto result = future.ExtractValue(); ev = std::make_unique<TEvPingResponse>(std::move(result)); - } catch (...) { + } catch (...) { ev = std::make_unique<TEvPingResponse>(TStringBuilder() << "Exception on ping response: " << CurrentExceptionMessage()); - } + } actorSystem->Send(new IEventHandle(selfId, selfId, ev.release(), 0, cookie)); } ); - } - + } + static constexpr ui64 ContinueLeaseRequestCookie = Max(); enum : ui64 { @@ -414,44 +414,44 @@ private: TConfig Config; - const TScope Scope; - const TString UserId; - const TString Id; - const TString OwnerId; - TPrivateClient Client; - - bool Requested = false; + const TScope Scope; + const TString UserId; + const TString Id; + const TString OwnerId; + TPrivateClient Client; + + bool Requested = false; TInstant StartLeaseTime; TRetryState RetryState; const TActorId Parent; const TInstant Deadline; - + std::deque<TForwardPingReqInfo> ForwardRequests; bool Finishing = false; bool FatalError = false; // Nonretryable error from PingTask or all retries finished. - TSchedulerCookieHolder SchedulerCookieHolder; -}; - -IActor* CreatePingerActor( - const TScope& scope, - const TString& userId, - const TString& id, - const TString& ownerId, + TSchedulerCookieHolder SchedulerCookieHolder; +}; + +IActor* CreatePingerActor( + const TScope& scope, + const TString& userId, + const TString& id, + const TString& ownerId, const TPrivateClient& client, const TActorId parent, const NConfig::TPingerConfig& config, const TInstant& deadline) -{ - return new TPingerActor( - scope, - userId, - id, - ownerId, +{ + return new TPingerActor( + scope, + userId, + id, + ownerId, client, parent, config, deadline); -} - +} + } /* NYq */ diff --git a/ydb/core/yq/libs/actors/proxy.cpp b/ydb/core/yq/libs/actors/proxy.cpp index a06fa41946..f06f2f8475 100644 --- a/ydb/core/yq/libs/actors/proxy.cpp +++ b/ydb/core/yq/libs/actors/proxy.cpp @@ -4,9 +4,9 @@ namespace NYq { using namespace NActors; -NActors::TActorId MakeYqlAnalyticsHttpProxyId() { - constexpr TStringBuf name = "YQLHTTPROXY"; - return NActors::TActorId(0, name); -} - +NActors::TActorId MakeYqlAnalyticsHttpProxyId() { + constexpr TStringBuf name = "YQLHTTPROXY"; + return NActors::TActorId(0, name); +} + } // namespace NYq diff --git a/ydb/core/yq/libs/actors/proxy.h b/ydb/core/yq/libs/actors/proxy.h index c4240d998a..442d29ac44 100644 --- a/ydb/core/yq/libs/actors/proxy.h +++ b/ydb/core/yq/libs/actors/proxy.h @@ -2,13 +2,13 @@ #include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include "run_actor_params.h" #include <util/datetime/base.h> - + #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/private_client/private_client.h> #include <ydb/core/yq/libs/shared_resources/db_pool.h> #include <ydb/core/yq/libs/shared_resources/shared_resources.h> #include <ydb/core/yq/libs/signer/signer.h> - + #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/worker_manager/interface/counters.h> @@ -16,10 +16,10 @@ #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/time_provider/time_provider.h> -#include <library/cpp/random_provider/random_provider.h> - +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/time_provider/time_provider.h> +#include <library/cpp/random_provider/random_provider.h> + #include <ydb/core/yq/libs/common/service_counters.h> namespace NKikimr { @@ -30,62 +30,62 @@ namespace NKikimr { namespace NYq { -NActors::TActorId MakeYqlAnalyticsHttpProxyId(); -NActors::TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId); - -NActors::IActor* CreatePendingFetcher( +NActors::TActorId MakeYqlAnalyticsHttpProxyId(); +NActors::TActorId MakeYqlAnalyticsFetcherId(ui32 nodeId); + +NActors::IActor* CreatePendingFetcher( const NYq::TYqSharedResources::TPtr& yqSharedResources, const ::NYq::NConfig::TCommonConfig& commonConfig, const ::NYq::NConfig::TCheckpointCoordinatorConfig& checkpointCoordinatorConfig, const ::NYq::NConfig::TPrivateApiConfig& privateApiConfig, const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<ITimeProvider> timeProvider, - TIntrusivePtr<IRandomProvider> randomProvider, - NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<ITimeProvider> timeProvider, + TIntrusivePtr<IRandomProvider> randomProvider, + NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory, const ::NYq::NCommon::TServiceCounters& serviceCounters, NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory, NYql::IHTTPGateway::TPtr s3Gateway, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const NMonitoring::TDynamicCounterPtr& clientCounters - ); - + ); + NActors::IActor* CreateRunActor( const ::NYq::NCommon::TServiceCounters& serviceCounters, TRunActorParams&& params ); -struct TResultId { - TString Id; - int SetId; +struct TResultId { + TString Id; + int SetId; TString HistoryId; TString Owner; TString CloudId; -}; - -NActors::IActor* CreateResultWriter( - const NYdb::TDriver& driver, +}; + +NActors::IActor* CreateResultWriter( + const NYdb::TDriver& driver, const NActors::TActorId& executerId, - const TString& resultType, + const TString& resultType, const NConfig::TPrivateApiConfig& privateApiConfig, - const TResultId& resultId, - const TVector<TString>& columns, + const TResultId& resultId, + const TVector<TString>& columns, const TString& traceId, const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters ); - + NActors::IActor* CreatePingerActor( - const TScope& scope, - const TString& userId, - const TString& id, - const TString& owner, + const TScope& scope, + const TString& userId, + const TString& id, + const TString& owner, const NYq::TPrivateClient& client, const NActors::TActorId parent, const NConfig::TPingerConfig& config, const TInstant& deadline); - -TString MakeInternalError(const TString& text); - + +TString MakeInternalError(const TString& text); + } /* NYq */ diff --git a/ydb/core/yq/libs/actors/result_writer.cpp b/ydb/core/yq/libs/actors/result_writer.cpp index 4f58d9acca..d51b8aae9f 100644 --- a/ydb/core/yq/libs/actors/result_writer.cpp +++ b/ydb/core/yq/libs/actors/result_writer.cpp @@ -1,21 +1,21 @@ #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> -#include "proxy.h" - +#include "proxy.h" + #include <ydb/core/protos/services.pb.h> #include <ydb/core/yq/libs/common/rows_proto_splitter.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <ydb/library/yql/providers/dq/actors/proto_builder.h> #include <ydb/library/yql/providers/dq/actors/events.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> #include <library/cpp/protobuf/interop/cast.h> - + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/control_plane_storage/events/events.h> #include <ydb/core/yq/libs/private_client/private_client.h> @@ -26,69 +26,69 @@ LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Writer: " << TraceId << ": " << stream) #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, "Writer: " << TraceId << ": " << stream) - + namespace NYq { - -using namespace NActors; -using namespace NYql; -using namespace NDqs; - -class TResultWriter : public NActors::TActorBootstrapped<TResultWriter> { -public: - TResultWriter( - const NYdb::TDriver& driver, + +using namespace NActors; +using namespace NYql; +using namespace NDqs; + +class TResultWriter : public NActors::TActorBootstrapped<TResultWriter> { +public: + TResultWriter( + const NYdb::TDriver& driver, const NActors::TActorId& executerId, - const TString& resultType, + const TString& resultType, const NConfig::TPrivateApiConfig& privateApiConfig, - const TResultId& resultId, - const TVector<TString>& columns, + const TResultId& resultId, + const TVector<TString>& columns, const TString& traceId, const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters) : ExecuterId(executerId) , ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns)) - , ResultId({resultId}) - , TraceId(traceId) + , ResultId({resultId}) + , TraceId(traceId) , Deadline(deadline) , Client( - driver, + driver, NYdb::TCommonClientSettings() .DiscoveryEndpoint(privateApiConfig.GetTaskServiceEndpoint()) .Database(privateApiConfig.GetTaskServiceDatabase() ? privateApiConfig.GetTaskServiceDatabase(): TMaybe<TString>()), clientCounters) - { } - + { } + static constexpr char ActorName[] = "YQ_RESULT_WRITER"; - + void Bootstrap(const TActorContext&) { - LOG_I("Bootstrap"); - Become(&TResultWriter::StateFunc); - } - -private: - STRICT_STFUNC(StateFunc, - cFunc(NActors::TEvents::TEvPoison::EventType, PassAway) - HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) - - HFunc(NDq::TEvDqCompute::TEvChannelData, OnChannelData) - HFunc(TEvReadyState, OnReadyState); - HFunc(TEvQueryResponse, OnQueryResult); + LOG_I("Bootstrap"); + Become(&TResultWriter::StateFunc); + } + +private: + STRICT_STFUNC(StateFunc, + cFunc(NActors::TEvents::TEvPoison::EventType, PassAway) + HFunc(NActors::TEvents::TEvUndelivered, OnUndelivered) + + HFunc(NDq::TEvDqCompute::TEvChannelData, OnChannelData) + HFunc(TEvReadyState, OnReadyState); + HFunc(TEvQueryResponse, OnQueryResult); hFunc(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse, HandleResponse); - ) - - void PassAway() { - auto duration = (TInstant::Now()-StartTime); + ) + + void PassAway() { + auto duration = (TInstant::Now()-StartTime); LOG_I("FinishWrite, Records: " << RowIndex << " HasError: " << HasError << " Size: " << Size << " Rows: " << Rows << " FreeSpace: " << FreeSpace << " Duration: " << duration << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); - NActors::IActor::PassAway(); - } - - void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext& ) { + NActors::IActor::PassAway(); + } + + void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&, const NActors::TActorContext& ) { auto req = MakeHolder<TEvDqFailure>(TIssue("Undelivered").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, /*needFallback=*/false); Send(ExecuterId, req.Release()); - HasError = true; - } - + HasError = true; + } + void SendIssuesAndSetErrorFlag(const TIssues& issues) { LOG_E("ControlPlane WriteResult Issues: " << issues.ToString()); Issues.AddIssues(issues); @@ -99,27 +99,27 @@ private: } void MaybeFinish() { - if (Finished && Requests.empty()) { + if (Finished && Requests.empty()) { Send(ExecuterId, new TEvGraphFinished()); - } - } - - void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { - Finished = true; + } + } + + void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { + Finished = true; NYql::NDqProto::TQueryResponse queryResult(ev->Get()->Record); *queryResult.MutableYson() = ResultBuilder->BuildYson(Head); if (!Issues.Empty()) { IssuesToMessage(Issues, queryResult.MutableIssues()); - } + } queryResult.SetTruncated(Truncated); queryResult.SetRowsCount(Rows); Send(ExecuterId, new TEvQueryResponse(std::move(queryResult))); - } - - void OnReadyState(TEvReadyState::TPtr&, const TActorContext&) { } - + } + + void OnReadyState(TEvReadyState::TPtr&, const TActorContext&) { } + void HandleResponse(NYq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev) { const auto& issues = ev->Get()->Issues; auto it = Requests.find(ev->Get()->RequestId); @@ -168,15 +168,15 @@ private: SendResult(); // Send remaining rows } - void StopChannel(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { - auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); - res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); - res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); - res->Record.SetFreeSpace(0); - res->Record.SetFinish(true); - Send(ev->Sender, res.Release()); - } - + void StopChannel(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { + auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); + res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); + res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); + res->Record.SetFreeSpace(0); + res->Record.SetFinish(true); + Send(ev->Sender, res.Release()); + } + Yq::Private::WriteTaskResultRequest CreateProtoRequestWithoutResultSet(ui64 startRowIndex) { Yq::Private::WriteTaskResultRequest protoReq; protoReq.set_owner_id(ResultId.Owner); @@ -238,12 +238,12 @@ private: Requests[Cookie].MessagesNum = splittedResultSets.ResultSets.size(); } - void ProcessData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { - auto& data = ev->Get()->Record.GetChannelData().GetData(); + void ProcessData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev) { + auto& data = ev->Get()->Record.GetChannelData().GetData(); auto resultSet = ResultBuilder->BuildResultSet({data}); - FreeSpace -= data.GetRaw().size(); + FreeSpace -= data.GetRaw().size(); OccupiedSpace += data.GetRaw().size(); - + if (OccupiedSpace > SpaceLimitPerQuery) { TIssues issues; issues.AddIssue(TStringBuilder() << "Can not write results with size > " << SpaceLimitPerQuery / (1024 * 1024) << "_MB"); @@ -254,39 +254,39 @@ private: ui64 startRowIndex = RowIndex; RowIndex += resultSet.rows().size(); - auto& request = Requests[Cookie]; - request.Sender = ev->Sender; - request.ChannelId = ev->Get()->Record.GetChannelData().GetChannelId(); - request.SeqNo = ev->Get()->Record.GetSeqNo(); - request.Size = data.GetRaw().size(); - + auto& request = Requests[Cookie]; + request.Sender = ev->Sender; + request.ChannelId = ev->Get()->Record.GetChannelData().GetChannelId(); + request.SeqNo = ev->Get()->Record.GetSeqNo(); + request.Size = data.GetRaw().size(); + ConstructResults(resultSet, startRowIndex); SendResult(); - if (!Truncated && - (!AllResultsBytesLimit || Size + data.GetRaw().size() < *AllResultsBytesLimit) - && (!RowsLimitPerWrite || Rows + data.GetRows() < *RowsLimitPerWrite)) { - Head.push_back(data); - } else { - Truncated = true; - } - - Size += data.GetRaw().size(); - Rows += data.GetRows(); - Cookie++; - } - - void OnChannelData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ctx); - - if (HasError) { - StopChannel(ev); - return; - } - - try { - if (ev->Get()->Record.GetChannelData().HasData()) { - ProcessData(ev); + if (!Truncated && + (!AllResultsBytesLimit || Size + data.GetRaw().size() < *AllResultsBytesLimit) + && (!RowsLimitPerWrite || Rows + data.GetRows() < *RowsLimitPerWrite)) { + Head.push_back(data); + } else { + Truncated = true; + } + + Size += data.GetRaw().size(); + Rows += data.GetRows(); + Cookie++; + } + + void OnChannelData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + + if (HasError) { + StopChannel(ev); + return; + } + + try { + if (ev->Get()->Record.GetChannelData().HasData()) { + ProcessData(ev); } else { auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); @@ -294,9 +294,9 @@ private: res->Record.SetFreeSpace(FreeSpace); res->Record.SetFinish(HasError); Send(ev->Sender, res.Release()); - + auto duration = (TInstant::Now()-StartTime); - + LOG_D("ChannelData, Records: " << RowIndex << " HasError: " << HasError << " Size: " << Size @@ -305,46 +305,46 @@ private: << " Duration: " << duration << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); } - } catch (...) { - LOG_E(CurrentExceptionMessage()); + } catch (...) { + LOG_E(CurrentExceptionMessage()); auto req = MakeHolder<TEvDqFailure>(TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR), /*retriable=*/ false, - /*needFallback=*/false); + /*needFallback=*/false); Send(ExecuterId, req.Release()); - HasError = true; - } - } - - struct TRequest { - TActorId Sender; - ui64 ChannelId; - ui64 SeqNo; - i64 Size; + HasError = true; + } + } + + struct TRequest { + TActorId Sender; + ui64 ChannelId; + ui64 SeqNo; + i64 Size; size_t MessagesNum; - }; - THashMap<ui64, TRequest> Requests; - + }; + THashMap<ui64, TRequest> Requests; + TVector<NYql::NDqProto::TData> Head; - bool Truncated = false; - TMaybe<ui64> AllResultsBytesLimit = 10000; - TMaybe<ui64> RowsLimitPerWrite = 1000; - ui64 Size = 0; - ui64 Rows = 0; - + bool Truncated = false; + TMaybe<ui64> AllResultsBytesLimit = 10000; + TMaybe<ui64> RowsLimitPerWrite = 1000; + ui64 Size = 0; + ui64 Rows = 0; + const TActorId ExecuterId; - THolder<TProtoBuilder> ResultBuilder; - const TResultId ResultId; - const TString TraceId; + THolder<TProtoBuilder> ResultBuilder; + const TResultId ResultId; + const TString TraceId; TInstant Deadline; TPrivateClient Client; - const TInstant StartTime = TInstant::Now(); - - ui64 RowIndex = 0; - ui64 Cookie = 0; - i64 FreeSpace = 64_MB; // TODO: make global free space + const TInstant StartTime = TInstant::Now(); + + ui64 RowIndex = 0; + ui64 Cookie = 0; + i64 FreeSpace = 64_MB; // TODO: make global free space const size_t ProtoMessageLimit = 10_MB; const size_t MaxRowsCountPerChunk = 100'000; - bool HasError = false; - bool Finished = false; + bool HasError = false; + bool Finished = false; NYql::TIssues Issues; ui64 SpaceLimitPerQuery = 20_MB; ui64 OccupiedSpace = 0; @@ -352,20 +352,20 @@ private: TVector<Yq::Private::WriteTaskResultRequest> ResultChunks; size_t CurChunkInd = 0; ui32 InflightCounter = 0; -}; - -NActors::IActor* CreateResultWriter( - const NYdb::TDriver& driver, +}; + +NActors::IActor* CreateResultWriter( + const NYdb::TDriver& driver, const NActors::TActorId& executerId, - const TString& resultType, + const TString& resultType, const NConfig::TPrivateApiConfig& privateApiConfig, - const TResultId& resultId, - const TVector<TString>& columns, + const TResultId& resultId, + const TVector<TString>& columns, const TString& traceId, const TInstant& deadline, const NMonitoring::TDynamicCounterPtr& clientCounters) -{ +{ return new TResultWriter(driver, executerId, resultType, privateApiConfig, resultId, columns, traceId, deadline, clientCounters); -} - +} + } // namespace NYq diff --git a/ydb/core/yq/libs/actors/run_actor.cpp b/ydb/core/yq/libs/actors/run_actor.cpp index d53199af79..5549f8f254 100644 --- a/ydb/core/yq/libs/actors/run_actor.cpp +++ b/ydb/core/yq/libs/actors/run_actor.cpp @@ -1,8 +1,8 @@ -#include "proxy.h" +#include "proxy.h" #include "clusters_from_connections.h" #include "system_clusters.h" #include "table_bindings_from_bindings.h" - + #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/dq/actors/compute/dq_checkpoints.h> #include <ydb/library/yql/dq/actors/dq.h> @@ -41,23 +41,23 @@ #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/public/issue/protos/issue_message.pb.h> - + #include <ydb/library/mkql_proto/mkql_proto.h> #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/log.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/log.h> #include <ydb/core/yq/libs/common/entity_id.h> #include <ydb/core/yq/libs/actors/nodes_manager.h> #include <ydb/core/yq/libs/gateway/empty_gateway.h> #include <ydb/core/yq/libs/read_rule/read_rule_creator.h> #include <ydb/core/yq/libs/read_rule/read_rule_deleter.h> -#include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> -#include <util/system/hostname.h> - +#include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> +#include <util/system/hostname.h> + #include <library/cpp/json/yson/json2yson.h> #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> @@ -74,16 +74,16 @@ #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, Params.QueryId << " RunActor : " << stream) - + #define LOG_D(stream) \ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, Params.QueryId << " RunActor : " << stream) - + namespace NYq { - -using namespace NActors; -using namespace NYql; + +using namespace NActors; +using namespace NYql; using namespace NDqs; - + class TDeferredCountersCleanupActor : public NActors::TActorBootstrapped<TDeferredCountersCleanupActor> { public: TDeferredCountersCleanupActor( @@ -135,30 +135,30 @@ public: , MaxTasksPerOperation(Params.CommonConfig.GetMaxTasksPerOperation() ? Params.CommonConfig.GetMaxTasksPerOperation() : 40) { } - + static constexpr char ActorName[] = "YQ_RUN_ACTOR"; - + void Bootstrap() { LOG_D("Start run actor. Compute state: " << YandexQuery::QueryMeta::ComputeStatus_Name(Params.Status)); LogReceivedParams(); - Pinger = Register( - CreatePingerActor( - Params.Scope, - Params.UserId, + Pinger = Register( + CreatePingerActor( + Params.Scope, + Params.UserId, Params.QueryId, - Params.Owner, - TPrivateClient( - Params.Driver, - NYdb::TCommonClientSettings() + Params.Owner, + TPrivateClient( + Params.Driver, + NYdb::TCommonClientSettings() .DiscoveryEndpoint(Params.PrivateApiConfig.GetTaskServiceEndpoint()) .EnableSsl(Params.PrivateApiConfig.GetSecureTaskService()) - .AuthToken(Params.AuthToken) + .AuthToken(Params.AuthToken) .Database(Params.PrivateApiConfig.GetTaskServiceDatabase() ? Params.PrivateApiConfig.GetTaskServiceDatabase() : TMaybe<TString>()), Params.ClientCounters), SelfId(), Params.PingerConfig, Params.Deadline - )); + )); Become(&TRunActor::StateFuncWrapper<&TRunActor::StateFunc>); try { Run(); @@ -166,7 +166,7 @@ public: FailOnException(); } } - + private: template <void (TRunActor::* DelegatedStateFunc)(STFUNC_SIG)> STFUNC(StateFuncWrapper) { @@ -186,8 +186,8 @@ private: hFunc(TEvents::TEvQueryActionResult, Handle); hFunc(TEvents::TEvForwardPingResponse, Handle); hFunc(TEvCheckpointCoordinator::TEvZeroCheckpointDone, Handle); - ) - + ) + STRICT_STFUNC(FinishStateFunc, hFunc(NYq::TEvents::TEvDataStreamsReadRulesCreationResult, HandleFinish); hFunc(TEvents::TEvDataStreamsReadRulesDeletionResult, HandleFinish); @@ -270,9 +270,9 @@ private: default: Abort("Fail to start query from unexpected status " + YandexQuery::QueryMeta::ComputeStatus_Name(Params.Status), YandexQuery::QueryMeta::FAILED); break; - } - } - + } + } + void HandleConnections() { LOG_D("HandleConnections"); @@ -285,8 +285,8 @@ private: Connections[connection.content().name()] = connection; // Necessary for TDatabaseAsyncResolverWithMeta YqConnections.emplace(connection.meta().id(), connection); } - } - + } + void RunProgram() { LOG_D("RunProgram"); if (!CompileQuery()) { @@ -308,8 +308,8 @@ private: if (!Finishing) { Abort("Internal Error", YandexQuery::QueryMeta::FAILED); return; - } - + } + // Already finishing. Fail instantly. Issues.AddIssue("Internal Error"); @@ -326,8 +326,8 @@ private: } SendPingAndPassAway(); - } - + } + void Handle(TEvents::TEvQueryActionResult::TPtr& ev) { Action = ev->Get()->Action; LOG_D("New query action received: " << YandexQuery::QueryAction_Name(Action)); @@ -903,7 +903,7 @@ private: *request.MutableSettings() = dqGraphParams.GetSettings(); *request.MutableSecureParams() = dqGraphParams.GetSecureParams(); *request.MutableColumns() = dqGraphParams.GetColumns(); - NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); + NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); NActors::TActivationContext::Send(new IEventHandle(ExecuterId, SelfId(), new NYql::NDqs::TEvGraphRequest(request, ControlId, resultId, CheckpointCoordinatorId))); LOG_D("Executer: " << ExecuterId << ", Controller: " << ControlId << ", ResultIdActor: " << resultId << ", CheckPointCoordinatior " << CheckpointCoordinatorId); } @@ -1113,9 +1113,9 @@ private: LOG_D("Compiling query ..."); NYql::TGatewaysConfig gatewaysConfig; SetupDqSettings(*gatewaysConfig.MutableDq()->MutableDefaultSettings()); - + THashMap<TString, TString> clusters; - + //todo: consider cluster name clashes AddClustersFromConfig(gatewaysConfig, clusters); AddSystemClusters(gatewaysConfig, clusters, Params.AuthToken); @@ -1138,15 +1138,15 @@ private: QueryStateUpdateRequest.set_owner_id(Params.Owner); dataProvidersInit.push_back(GetDqDataProviderInitializer(&CreateInMemoryExecTransformer, NYq::CreateEmptyGateway(SelfId()), Params.DqCompFactory, {}, nullptr)); } - + { dataProvidersInit.push_back(GetYdbDataProviderInitializer(Params.Driver, Params.CredentialsFactory, dbResolver)); } - + { dataProvidersInit.push_back(GetClickHouseDataProviderInitializer(Params.S3Gateway, dbResolver)); } - + { dataProvidersInit.push_back(GetS3DataProviderInitializer(Params.S3Gateway, Params.CredentialsFactory)); } @@ -1173,14 +1173,14 @@ private: progFactory.SetModules(Params.ModuleResolver); progFactory.SetUdfResolver(NYql::NCommon::CreateSimpleUdfResolver(Params.FunctionRegistry, nullptr)); progFactory.SetGatewaysConfig(&gatewaysConfig); - + SessionId = TStringBuilder() << Params.QueryId << '#' << Params.ResultId << '#' << Params.Scope.ToString() << '#' << Params.Owner << '#' << Params.CloudId; - + Program = progFactory.Create("-stdin-", Params.Sql, SessionId); Program->EnableResultPosition(); @@ -1207,7 +1207,7 @@ private: if (Params.ExecuteMode == YandexQuery::ExecuteMode::PARSE) { return true; } - } + } // compile phase { @@ -1237,7 +1237,7 @@ private: Issues.AddIssue(TStringBuilder() << "Unexpected execute mode " << static_cast<int>(Params.ExecuteMode)); return false; } - + futureStatus.Subscribe([actorSystem = NActors::TActivationContext::ActorSystem(), selfId = SelfId()](const TProgram::TFutureStatus& f) { actorSystem->Send(selfId, new TEvents::TEvAsyncContinue(f)); }); @@ -1270,11 +1270,11 @@ private: Abort(TStringBuilder() << "Run query failed: " << ToString(status), YandexQuery::QueryMeta::FAILED, Program->Issues()); } } - + void Handle(NActors::TEvents::TEvUndelivered::TPtr&) { Fail("TRunActor::OnUndelivered"); } - + TString FindTokenByName(const TString& tokenName) const { for (auto& graphParams : DqGraphParams) { const auto& secureParams = graphParams.GetSecureParams(); @@ -1318,13 +1318,13 @@ private: private: TRunActorParams Params; THashMap<TString, YandexQuery::Connection> YqConnections; - + TProgramPtr Program; TIssues Issues; TIssues TransientIssues; TQueryResult QueryResult; TInstant Deadline; - TActorId Pinger; + TActorId Pinger; TInstant CreatedAt; YandexQuery::QueryAction Action = YandexQuery::QueryAction::QUERY_ACTION_UNSPECIFIED; std::vector<NYq::NProto::TGraphParams> DqGraphParams; @@ -1373,6 +1373,6 @@ IActor* CreateRunActor( TRunActorParams&& params ) { return new TRunActor(serviceCounters, std::move(params)); -} - +} + } /* NYq */ diff --git a/ydb/core/yq/libs/actors/run_actor_params.cpp b/ydb/core/yq/libs/actors/run_actor_params.cpp index 57dbb0ea87..1446cae1bb 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.cpp +++ b/ydb/core/yq/libs/actors/run_actor_params.cpp @@ -19,11 +19,11 @@ TRunActorParams::TRunActorParams( const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, const TString& sql, - const TScope& scope, + const TScope& scope, const TString& authToken, - const TActorId& databaseResolver, + const TActorId& databaseResolver, const TString& queryId, - const TString& userId, + const TString& userId, const TString& owner, const int64_t previousQueryRevision, TVector<YandexQuery::Connection> connections, @@ -60,12 +60,12 @@ TRunActorParams::TRunActorParams( , GatewaysConfig(gatewaysConfig) , PingerConfig(pingerConfig) , Sql(sql) - , Scope(scope) + , Scope(scope) , AuthToken(authToken) - , DatabaseResolver(databaseResolver) + , DatabaseResolver(databaseResolver) , QueryId(queryId) - , UserId(userId) - , Owner(owner) + , UserId(userId) + , Owner(owner) , PreviousQueryRevision(previousQueryRevision) , Connections(std::move(connections)) , Bindings(std::move(bindings)) diff --git a/ydb/core/yq/libs/actors/run_actor_params.h b/ydb/core/yq/libs/actors/run_actor_params.h index 431014018b..bad5c3c32b 100644 --- a/ydb/core/yq/libs/actors/run_actor_params.h +++ b/ydb/core/yq/libs/actors/run_actor_params.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <ydb/core/yq/libs/config/protos/common.pb.h> #include <ydb/core/yq/libs/config/protos/pinger.pb.h> #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> @@ -33,11 +33,11 @@ struct TRunActorParams { // TODO2 : Change name const ::NYq::NConfig::TGatewaysConfig& gatewaysConfig, const ::NYq::NConfig::TPingerConfig& pingerConfig, const TString& sql, - const TScope& scope, + const TScope& scope, const TString& authToken, const NActors::TActorId& databaseResolver, const TString& queryId, - const TString& userId, + const TString& userId, const TString& owner, const int64_t previousQueryRevision, TVector<YandexQuery::Connection> connections, @@ -79,12 +79,12 @@ struct TRunActorParams { // TODO2 : Change name const ::NYq::NConfig::TGatewaysConfig GatewaysConfig; const ::NYq::NConfig::TPingerConfig PingerConfig; const TString Sql; - const TScope Scope; + const TScope Scope; const TString AuthToken; const NActors::TActorId DatabaseResolver; const TString QueryId; - const TString UserId; - const TString Owner; + const TString UserId; + const TString Owner; const int64_t PreviousQueryRevision; const TVector<YandexQuery::Connection> Connections; const TVector<YandexQuery::Binding> Bindings; diff --git a/ydb/core/yq/libs/actors/task_ping.cpp b/ydb/core/yq/libs/actors/task_ping.cpp index 271af322bc..29410dccd1 100644 --- a/ydb/core/yq/libs/actors/task_ping.cpp +++ b/ydb/core/yq/libs/actors/task_ping.cpp @@ -77,7 +77,7 @@ public: Become(&TTaskPingRequestActor::StateFunc); const auto& req = Ev->Record; OperationId = req.query_id().value(); - OwnerId = req.owner_id(); + OwnerId = req.owner_id(); Scope = req.scope(); Deadline = NProtoInterop::CastFromProto(req.deadline()); LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes"); @@ -223,7 +223,7 @@ private: const TInstant StartTime; TString OperationId; - TString OwnerId; + TString OwnerId; TString Scope; TInstant Deadline; NYql::TIssues Issues; diff --git a/ydb/core/yq/libs/actors/ya.make b/ydb/core/yq/libs/actors/ya.make index e6af9262ee..3638593814 100644 --- a/ydb/core/yq/libs/actors/ya.make +++ b/ydb/core/yq/libs/actors/ya.make @@ -4,7 +4,7 @@ LIBRARY() SRCS( clusters_from_connections.cpp - database_resolver.cpp + database_resolver.cpp error.cpp nodes_health_check.cpp nodes_manager.cpp @@ -13,7 +13,7 @@ SRCS( proxy.cpp proxy_private.cpp result_writer.cpp - run_actor.cpp + run_actor.cpp run_actor_params.cpp system_clusters.cpp table_bindings_from_bindings.cpp diff --git a/ydb/core/yq/libs/common/cache.h b/ydb/core/yq/libs/common/cache.h index a5c9d10fa9..e79eb32839 100644 --- a/ydb/core/yq/libs/common/cache.h +++ b/ydb/core/yq/libs/common/cache.h @@ -1,116 +1,116 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/datetime/base.h> -#include <util/system/mutex.h> - +#pragma once + +#include <util/generic/string.h> +#include <util/datetime/base.h> +#include <util/system/mutex.h> + namespace NYq { - -struct TTtlCacheSettings { - TDuration Ttl = TDuration::Minutes(10); - TDuration ErrorTtl = TDuration::Minutes(10); - ui64 MaxSize = 1000000; - bool TouchOnGet = false; - - TTtlCacheSettings& SetTtl(TDuration ttl) { - Ttl = ttl; - return *this; - } - - TTtlCacheSettings& SetErrorTtl(TDuration ttl) { - ErrorTtl = ttl; - return *this; - } - - TTtlCacheSettings& SetMaxSize(ui64 maxSize) { - MaxSize = maxSize; - return *this; - } - - TTtlCacheSettings& SetTouchOnGet(bool flag) { - TouchOnGet = flag; - return *this; - } -}; - -template<typename TKey, typename TValue> -class TTtlCache { -public: - TTtlCache(const TTtlCacheSettings& config = TTtlCacheSettings()) - : Config(config) - { } - - void Put(const TKey& key, const TMaybe<TValue>& value) { - TGuard<TMutex> lock(Mutex); - auto it = Data.find(key); - if (it != Data.end()) { - it->second.Queue->erase(it->second.Position); - } - - auto* queue = value.Empty() - ? &ErrorQueue - : &OkQueue; - queue->push_back({key, TInstant::Now()}); - auto position = queue->end(); - --position; - TCacheObject object = {value, position, queue}; - Data[key] = object; - DropOld(lock); - } - - bool Get(const TKey& key, TMaybe<TValue>* value) { - TGuard<TMutex> lock(Mutex); - auto it = Data.find(key); - if (it == Data.end()) { - return false; - } - *value = it->second.Endpoint; - - if (Config.TouchOnGet) { - it->second.Queue->erase(it->second.Position); - it->second.Queue->push_back({key, TInstant::Now()}); - it->second.Position = it->second.Queue->end(); - --it->second.Position; - } - - return true; - } - - ui64 Size() const { - return Data.size(); - } - -private: - void DropOld(const TGuard<TMutex>& ) { - auto now = TInstant::Now(); - auto cleanUp = [&](TList<TKeyAndTime>& queue, const TDuration& ttl) { - for (auto it = queue.begin(); it != queue.end() && (it->LastAccess <= now - ttl || Data.size() > Config.MaxSize) ; ) { - Data.erase(it->Key); - it = queue.erase(it); - } - }; - - cleanUp(ErrorQueue, Config.ErrorTtl); - cleanUp(OkQueue, Config.Ttl); - } - - TMutex Mutex; - - struct TKeyAndTime { - TKey Key; - TInstant LastAccess; - }; - TList<TKeyAndTime> OkQueue; - TList<TKeyAndTime> ErrorQueue; - - struct TCacheObject { - TMaybe<TValue> Endpoint; - typename TList<TKeyAndTime>::iterator Position; - TList<TKeyAndTime>* Queue; - }; - THashMap<TKey, TCacheObject> Data; - - TTtlCacheSettings Config; -}; - + +struct TTtlCacheSettings { + TDuration Ttl = TDuration::Minutes(10); + TDuration ErrorTtl = TDuration::Minutes(10); + ui64 MaxSize = 1000000; + bool TouchOnGet = false; + + TTtlCacheSettings& SetTtl(TDuration ttl) { + Ttl = ttl; + return *this; + } + + TTtlCacheSettings& SetErrorTtl(TDuration ttl) { + ErrorTtl = ttl; + return *this; + } + + TTtlCacheSettings& SetMaxSize(ui64 maxSize) { + MaxSize = maxSize; + return *this; + } + + TTtlCacheSettings& SetTouchOnGet(bool flag) { + TouchOnGet = flag; + return *this; + } +}; + +template<typename TKey, typename TValue> +class TTtlCache { +public: + TTtlCache(const TTtlCacheSettings& config = TTtlCacheSettings()) + : Config(config) + { } + + void Put(const TKey& key, const TMaybe<TValue>& value) { + TGuard<TMutex> lock(Mutex); + auto it = Data.find(key); + if (it != Data.end()) { + it->second.Queue->erase(it->second.Position); + } + + auto* queue = value.Empty() + ? &ErrorQueue + : &OkQueue; + queue->push_back({key, TInstant::Now()}); + auto position = queue->end(); + --position; + TCacheObject object = {value, position, queue}; + Data[key] = object; + DropOld(lock); + } + + bool Get(const TKey& key, TMaybe<TValue>* value) { + TGuard<TMutex> lock(Mutex); + auto it = Data.find(key); + if (it == Data.end()) { + return false; + } + *value = it->second.Endpoint; + + if (Config.TouchOnGet) { + it->second.Queue->erase(it->second.Position); + it->second.Queue->push_back({key, TInstant::Now()}); + it->second.Position = it->second.Queue->end(); + --it->second.Position; + } + + return true; + } + + ui64 Size() const { + return Data.size(); + } + +private: + void DropOld(const TGuard<TMutex>& ) { + auto now = TInstant::Now(); + auto cleanUp = [&](TList<TKeyAndTime>& queue, const TDuration& ttl) { + for (auto it = queue.begin(); it != queue.end() && (it->LastAccess <= now - ttl || Data.size() > Config.MaxSize) ; ) { + Data.erase(it->Key); + it = queue.erase(it); + } + }; + + cleanUp(ErrorQueue, Config.ErrorTtl); + cleanUp(OkQueue, Config.Ttl); + } + + TMutex Mutex; + + struct TKeyAndTime { + TKey Key; + TInstant LastAccess; + }; + TList<TKeyAndTime> OkQueue; + TList<TKeyAndTime> ErrorQueue; + + struct TCacheObject { + TMaybe<TValue> Endpoint; + typename TList<TKeyAndTime>::iterator Position; + TList<TKeyAndTime>* Queue; + }; + THashMap<TKey, TCacheObject> Data; + + TTtlCacheSettings Config; +}; + } // namespace NYq diff --git a/ydb/core/yq/libs/common/cache_ut.cpp b/ydb/core/yq/libs/common/cache_ut.cpp index 965d9ad270..4e5ace033d 100644 --- a/ydb/core/yq/libs/common/cache_ut.cpp +++ b/ydb/core/yq/libs/common/cache_ut.cpp @@ -1,58 +1,58 @@ -#include "cache.h" - +#include "cache.h" + #include <ydb/services/ydb/ydb_common_ut.h> - + using namespace NYq; - -using TCache = TTtlCache<int,int>; - -Y_UNIT_TEST_SUITE(Cache) { - Y_UNIT_TEST(Test1) { - TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(10)).SetMaxSize(100)); - TMaybe<int> t; - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 0); - UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), false); - cache.Put(100, 10); - UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), true); - UNIT_ASSERT_VALUES_EQUAL(*t, 10); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 1); - cache.Put(100, TMaybe<int>()); - UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), true); - UNIT_ASSERT_VALUES_EQUAL(t, TMaybe<int>()); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 1); - cache.Put(101, 10); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 2); - } - - - Y_UNIT_TEST(Test2) { - TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(10)).SetMaxSize(3)); - TMaybe<int> t; - cache.Put(100, 10); - cache.Put(101, 11); - cache.Put(102, 12); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); - cache.Put(103, 13); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); - UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), false); - } - - - Y_UNIT_TEST(Test3) { - TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(0)).SetMaxSize(3)); - cache.Put(100, 10); - cache.Put(101, 11); - cache.Put(102, 12); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 0); - } - - Y_UNIT_TEST(Test4) { - TCache cache(TTtlCacheSettings().SetErrorTtl(TDuration::Minutes(0)).SetMaxSize(3)); - cache.Put(100, 10); - cache.Put(101, 11); - cache.Put(102, 12); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); - cache.Put(102, TMaybe<int>()); - UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 2); - } -} + +using TCache = TTtlCache<int,int>; + +Y_UNIT_TEST_SUITE(Cache) { + Y_UNIT_TEST(Test1) { + TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(10)).SetMaxSize(100)); + TMaybe<int> t; + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 0); + UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), false); + cache.Put(100, 10); + UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), true); + UNIT_ASSERT_VALUES_EQUAL(*t, 10); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 1); + cache.Put(100, TMaybe<int>()); + UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), true); + UNIT_ASSERT_VALUES_EQUAL(t, TMaybe<int>()); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 1); + cache.Put(101, 10); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 2); + } + + + Y_UNIT_TEST(Test2) { + TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(10)).SetMaxSize(3)); + TMaybe<int> t; + cache.Put(100, 10); + cache.Put(101, 11); + cache.Put(102, 12); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); + cache.Put(103, 13); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); + UNIT_ASSERT_VALUES_EQUAL(cache.Get(100, &t), false); + } + + + Y_UNIT_TEST(Test3) { + TCache cache(TTtlCacheSettings().SetTtl(TDuration::Minutes(0)).SetMaxSize(3)); + cache.Put(100, 10); + cache.Put(101, 11); + cache.Put(102, 12); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 0); + } + + Y_UNIT_TEST(Test4) { + TCache cache(TTtlCacheSettings().SetErrorTtl(TDuration::Minutes(0)).SetMaxSize(3)); + cache.Put(100, 10); + cache.Put(101, 11); + cache.Put(102, 12); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 3); + cache.Put(102, TMaybe<int>()); + UNIT_ASSERT_VALUES_EQUAL(cache.Size(), 2); + } +} diff --git a/ydb/core/yq/libs/common/entity_id.cpp b/ydb/core/yq/libs/common/entity_id.cpp index 54d9ea3585..dab1d1abca 100644 --- a/ydb/core/yq/libs/common/entity_id.cpp +++ b/ydb/core/yq/libs/common/entity_id.cpp @@ -1,22 +1,22 @@ -#include "entity_id.h" +#include "entity_id.h" #include <util/stream/str.h> #include <util/system/yassert.h> -#include <util/datetime/base.h> -#include <util/system/unaligned_mem.h> -#include <util/random/easy.h> -#include <util/stream/format.h> -#include <util/string/ascii.h> - +#include <util/datetime/base.h> +#include <util/system/unaligned_mem.h> +#include <util/random/easy.h> +#include <util/stream/format.h> +#include <util/string/ascii.h> + namespace NYq { - -// used ascii order: IntToChar[i] < IntToChar[i+1] -constexpr char IntToChar[] = { + +// used ascii order: IntToChar[i] < IntToChar[i+1] +constexpr char IntToChar[] = { '0', '1', '2', '3', '4', '5', '6', '7', // 8 '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', // 16 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', // 24 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', // 32 -}; - +}; + static bool IsValidSymbol(char symbol) { return '0' <= symbol && symbol <= '9' || 'a' <= symbol && symbol <= 'v'; } @@ -33,14 +33,14 @@ TString Conv(ui32 n) { // 55 bits = 11 symbols TString ConvTime(ui64 time) { - char buf[12] = {0}; - for (int i = 0; i < 11; i++) { + char buf[12] = {0}; + for (int i = 0; i < 11; i++) { buf[10 - i] = IntToChar[time & 0x1f]; time >>= 5; - } - return buf; -} - + } + return buf; +} + /* - prefix [2 symbols (10 bit)] - service identifier - type [1 symbol (5 bit)] - resource type (query, job, connection, binding, result, ...) @@ -54,7 +54,7 @@ TString GetEntityIdAsString(const TString& prefix, EEntityType type) { Y_VERIFY(IsValidSymbol(prefix[1]), "not valid character for prefix[1]"); return GetEntityIdAsString(prefix, type, TInstant::Now(), RandomNumber<ui32>()); } - + TString GetEntityIdAsString(const TString& prefix, EEntityType type, TInstant now, ui32 rnd) { // The maximum time that fits in 55 bits. Necessary for the correct inversion of time static constexpr TInstant MaxSeconds = TInstant::FromValue(0x7FFFFFFFFFFFFFULL); @@ -63,8 +63,8 @@ TString GetEntityIdAsString(const TString& prefix, EEntityType type, TInstant no TStringStream stream; stream << prefix << static_cast<char>(type) << ConvTime(reversedTime) << Conv(rnd); return stream.Str(); -} - +} + struct TEntityIdGenerator : public IEntityIdGenerator { TEntityIdGenerator(const TString& prefix) : Prefix(prefix) diff --git a/ydb/core/yq/libs/common/entity_id.h b/ydb/core/yq/libs/common/entity_id.h index 5955184236..d34dee0f55 100644 --- a/ydb/core/yq/libs/common/entity_id.h +++ b/ydb/core/yq/libs/common/entity_id.h @@ -1,10 +1,10 @@ -#pragma once - -#include <util/generic/string.h> +#pragma once + +#include <util/generic/string.h> #include <util/datetime/base.h> - + namespace NYq { - + enum class EEntityType : char { UNDEFINED = 'u', QUERY = 'q', @@ -14,7 +14,7 @@ enum class EEntityType : char { BINDING = 'b', CHECKPOINT_GRAPH_DESCRIPTION = 'g', }; - + TString GetEntityIdAsString(const TString& prefix, EEntityType type); TString GetEntityIdAsString(const TString& prefix, EEntityType type, TInstant now, ui32 rnd); diff --git a/ydb/core/yq/libs/common/entity_id_ut.cpp b/ydb/core/yq/libs/common/entity_id_ut.cpp index e5b92b2bf6..8533bdcca0 100644 --- a/ydb/core/yq/libs/common/entity_id_ut.cpp +++ b/ydb/core/yq/libs/common/entity_id_ut.cpp @@ -1,41 +1,41 @@ -#include "entity_id.h" - +#include "entity_id.h" + #include <ydb/services/ydb/ydb_common_ut.h> #include <limits> - + using namespace NYq; - -Y_UNIT_TEST_SUITE(EntityId) { - Y_UNIT_TEST(Distinct) { - int count = 100; - TVector<TString> values; - THashSet<TString> uniq; - values.reserve(count); - for (int i = 0; i < count; i++) { + +Y_UNIT_TEST_SUITE(EntityId) { + Y_UNIT_TEST(Distinct) { + int count = 100; + TVector<TString> values; + THashSet<TString> uniq; + values.reserve(count); + for (int i = 0; i < count; i++) { values.push_back(GetEntityIdAsString("de", EEntityType::UNDEFINED)); - uniq.insert(values.back()); - } - - UNIT_ASSERT_VALUES_EQUAL(uniq.size(), values.size()); - } - - Y_UNIT_TEST(Order) { - int count = 100; - TVector<TString> values, copy; - values.reserve(count); - for (int i = 0; i < count; i++) { + uniq.insert(values.back()); + } + + UNIT_ASSERT_VALUES_EQUAL(uniq.size(), values.size()); + } + + Y_UNIT_TEST(Order) { + int count = 100; + TVector<TString> values, copy; + values.reserve(count); + for (int i = 0; i < count; i++) { values.push_back(GetEntityIdAsString("de", EEntityType::UNDEFINED)); - Sleep(TDuration::MilliSeconds(1)); // timer resolution - } - copy = values; - std::sort(copy.begin(), copy.end(), [](const auto& a, const auto& b) { - return a>b; - }); - - for (int i = 0; i < count; i++) { - UNIT_ASSERT_VALUES_EQUAL(values[i], copy[i]); - } - } + Sleep(TDuration::MilliSeconds(1)); // timer resolution + } + copy = values; + std::sort(copy.begin(), copy.end(), [](const auto& a, const auto& b) { + return a>b; + }); + + for (int i = 0; i < count; i++) { + UNIT_ASSERT_VALUES_EQUAL(values[i], copy[i]); + } + } Y_UNIT_TEST(MinId) { UNIT_ASSERT_VALUES_EQUAL(GetEntityIdAsString("00", EEntityType::UNDEFINED, TInstant::FromValue(0x7FFFFFFFFFFFFFULL), 0), "00u00000000000000000"); @@ -48,4 +48,4 @@ Y_UNIT_TEST_SUITE(EntityId) { Y_UNIT_TEST(CheckId) { UNIT_ASSERT_VALUES_EQUAL(GetEntityIdAsString("ut", EEntityType::QUERY, TInstant::ParseIso8601("2021-12-04"), 666666), "utquhdn55pa7vv00kb1a"); } -} +} diff --git a/ydb/core/yq/libs/common/ut/ya.make b/ydb/core/yq/libs/common/ut/ya.make index abdcd0533f..320ef17e66 100644 --- a/ydb/core/yq/libs/common/ut/ya.make +++ b/ydb/core/yq/libs/common/ut/ya.make @@ -1,27 +1,27 @@ UNITTEST_FOR(ydb/core/yq/libs/common) - + OWNER( g:kikimr g:yq ) -FORK_SUBTESTS() - -IF (SANITIZER_TYPE OR WITH_VALGRIND) - SIZE(MEDIUM) -ENDIF() - -SRCS( - cache_ut.cpp - entity_id_ut.cpp +FORK_SUBTESTS() + +IF (SANITIZER_TYPE OR WITH_VALGRIND) + SIZE(MEDIUM) +ENDIF() + +SRCS( + cache_ut.cpp + entity_id_ut.cpp rows_proto_splitter_ut.cpp -) - -PEERDIR( +) + +PEERDIR( ydb/library/yql/public/udf/service/stub ydb/services/ydb -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/common/ya.make b/ydb/core/yq/libs/common/ya.make index e14cb7a1da..4622695700 100644 --- a/ydb/core/yq/libs/common/ya.make +++ b/ydb/core/yq/libs/common/ya.make @@ -4,7 +4,7 @@ LIBRARY() SRCS( database_token_builder.cpp - entity_id.cpp + entity_id.cpp rows_proto_splitter.cpp ) @@ -19,8 +19,8 @@ PEERDIR( YQL_LAST_ABI_VERSION() -END() - +END() + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/yq/libs/db_schema/db_schema.cpp b/ydb/core/yq/libs/db_schema/db_schema.cpp index 916fb9e24d..fdda16df40 100644 --- a/ydb/core/yq/libs/db_schema/db_schema.cpp +++ b/ydb/core/yq/libs/db_schema/db_schema.cpp @@ -5,31 +5,31 @@ namespace NYq { -using namespace NYdb; - +using namespace NYdb; + TSqlQueryBuilder::TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName) - : TablePrefix(tablePrefix) + : TablePrefix(tablePrefix) , QueryName(queryName) -{ } - -TSqlQueryBuilder& TSqlQueryBuilder::PushPk() { - PkStack.emplace_back(Pk); - Pk.clear(); - return *this; -} - -TSqlQueryBuilder& TSqlQueryBuilder::PopPk() { - Pk = PkStack.back(); - PkStack.pop_back(); - return *this; -} - -TSqlQueryBuilder& TSqlQueryBuilder::AddPk(const TString& name, const TValue& value) -{ - Pk.push_back(name); +{ } + +TSqlQueryBuilder& TSqlQueryBuilder::PushPk() { + PkStack.emplace_back(Pk); + Pk.clear(); + return *this; +} + +TSqlQueryBuilder& TSqlQueryBuilder::PopPk() { + Pk = PkStack.back(); + PkStack.pop_back(); + return *this; +} + +TSqlQueryBuilder& TSqlQueryBuilder::AddPk(const TString& name, const TValue& value) +{ + Pk.push_back(name); return AddValue(name, value, "=="); -} - +} + TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const NYdb::TValue& value) { const auto typeStr = value.GetType().ToString(); @@ -44,51 +44,51 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const NYdb::TV return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const TValue& value, const TString& pred) -{ - TStringBuilder var; var << name; - if (Op) { - var << Counter; - } - auto typeStr = value.GetType().ToString(); - auto it = Parameters.find(var); - if (it == Parameters.end()) { - Parameters.emplace(var, typeStr); - ParametersOrdered.emplace_back(var, typeStr); - ParamsBuilder.AddParam("$" + var, value); - if (Op) { - Fields.emplace_back(name, var, pred); - } - } else if (it->second != typeStr) { - ythrow yexception() << "Parameter `" << name << "' type " << typeStr << " != " << it->second; - } - return *this; -} - +TSqlQueryBuilder& TSqlQueryBuilder::AddValue(const TString& name, const TValue& value, const TString& pred) +{ + TStringBuilder var; var << name; + if (Op) { + var << Counter; + } + auto typeStr = value.GetType().ToString(); + auto it = Parameters.find(var); + if (it == Parameters.end()) { + Parameters.emplace(var, typeStr); + ParametersOrdered.emplace_back(var, typeStr); + ParamsBuilder.AddParam("$" + var, value); + if (Op) { + Fields.emplace_back(name, var, pred); + } + } else if (it->second != typeStr) { + ythrow yexception() << "Parameter `" << name << "' type " << typeStr << " != " << it->second; + } + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::AddColNames(TVector<TString>&& colNames) { ColNames = colNames; return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::MaybeAddValue(const TString& name, const TMaybe<TValue>& value) { - if (value) { +TSqlQueryBuilder& TSqlQueryBuilder::MaybeAddValue(const TString& name, const TMaybe<TValue>& value) { + if (value) { return AddValue(name, *value, "=="); - } else { - return *this; - } -} - -TSqlQueryBuilder& TSqlQueryBuilder::AddText(const TString& sql, const TString& tableName) -{ - if (tableName) { + } else { + return *this; + } +} + +TSqlQueryBuilder& TSqlQueryBuilder::AddText(const TString& sql, const TString& tableName) +{ + if (tableName) { Text << SubstGlobalCopy(sql, TString("{TABLENAME}"), tableName); - } else { - Text << sql; - } - return *this; -} - + } else { + Text << sql; + } + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TString& tableName ) { if (tableName) { BeforeQuery << SubstGlobalCopy(sql, TString("{TABLENAME}"), tableName); @@ -99,35 +99,35 @@ TSqlQueryBuilder& TSqlQueryBuilder::AddBeforeQuery(const TString& sql, const TSt } -TSqlQueryBuilder& TSqlQueryBuilder::Upsert(const TString& tableName) { - Op = "UPSERT"; - Table = tableName; - return *this; -} - +TSqlQueryBuilder& TSqlQueryBuilder::Upsert(const TString& tableName) { + Op = "UPSERT"; + Table = tableName; + return *this; +} + TSqlQueryBuilder& TSqlQueryBuilder::Update(const TString& tableName) { Op = "UPDATE"; Table = tableName; return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::Insert(const TString& tableName) { - Op = "INSERT"; - Table = tableName; - return *this; -} - -TSqlQueryBuilder& TSqlQueryBuilder::Delete(const TString& tableName, int limit) { - Op = "DELETE"; - Table = tableName; - Limit = limit; - return *this; -} - -TSqlQueryBuilder& TSqlQueryBuilder::Get(const TString& tableName, int limit) { +TSqlQueryBuilder& TSqlQueryBuilder::Insert(const TString& tableName) { + Op = "INSERT"; + Table = tableName; + return *this; +} + +TSqlQueryBuilder& TSqlQueryBuilder::Delete(const TString& tableName, int limit) { + Op = "DELETE"; + Table = tableName; + Limit = limit; + return *this; +} + +TSqlQueryBuilder& TSqlQueryBuilder::Get(const TString& tableName, int limit) { Op = "SELECT"; Table = tableName; - Limit = limit; + Limit = limit; return *this; } @@ -137,11 +137,11 @@ TSqlQueryBuilder& TSqlQueryBuilder::GetCount(const TString& tableName) { return *this; } -TSqlQueryBuilder& TSqlQueryBuilder::GetLastDeleted() { - Op = "SELECT LAST DELETED"; - return *this; -} - +TSqlQueryBuilder& TSqlQueryBuilder::GetLastDeleted() { + Op = "SELECT LAST DELETED"; + return *this; +} + void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) { text << "WHERE "; size_t i = 0; @@ -153,12 +153,12 @@ void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) { } } i = 0; - for (const auto& t : Fields) { - auto k = std::get<0>(t); // structured capture don't work on win32 - auto v = std::get<1>(t); // structured capture don't work on win32 - auto p = std::get<2>(t); // structured capture don't work on win32 - - text << "`" << k << "` " << p << " " << "$" << v; + for (const auto& t : Fields) { + auto k = std::get<0>(t); // structured capture don't work on win32 + auto v = std::get<1>(t); // structured capture don't work on win32 + auto p = std::get<2>(t); // structured capture don't work on win32 + + text << "`" << k << "` " << p << " " << "$" << v; ++i; if (i != Fields.size()) { text << " and "; @@ -166,31 +166,31 @@ void TSqlQueryBuilder::ConstructWhereFilter(TStringBuilder& text) { } } -TSqlQueryBuilder& TSqlQueryBuilder::End() -{ - if (!Op) { - ythrow yexception() << "Empty operation"; - } - TStringBuilder text; - - if (Op == "DELETE") { +TSqlQueryBuilder& TSqlQueryBuilder::End() +{ + if (!Op) { + ythrow yexception() << "Empty operation"; + } + TStringBuilder text; + + if (Op == "DELETE") { text << "$todelete" << Counter << " = (SELECT * FROM `" << Table << "`\n"; - if (Pk.empty() && Fields.empty() && !Limit) { - ythrow yexception() << "dangerous sql query"; - } - if (!Pk.empty() || !Fields.empty()) { - ConstructWhereFilter(text); - } - if (Limit > 0) { - text << " LIMIT " << Limit; - } - text << ");\n"; - BeforeQuery << text; - - text.clear(); - + if (Pk.empty() && Fields.empty() && !Limit) { + ythrow yexception() << "dangerous sql query"; + } + if (!Pk.empty() || !Fields.empty()) { + ConstructWhereFilter(text); + } + if (Limit > 0) { + text << " LIMIT " << Limit; + } + text << ");\n"; + BeforeQuery << text; + + text.clear(); + text << "DELETE FROM `" << Table << "`\n"; - text << "ON (SELECT * from $todelete" << Counter << ");\n"; + text << "ON (SELECT * from $todelete" << Counter << ");\n"; } else if (Op == "SELECT") { text << Op << ' '; size_t i = 0; @@ -203,21 +203,21 @@ TSqlQueryBuilder& TSqlQueryBuilder::End() } text << "\nFROM `" << Table << "`\n"; ConstructWhereFilter(text); - if (Limit > 0) { - text << " LIMIT " << Limit; - } + if (Limit > 0) { + text << " LIMIT " << Limit; + } text << ";\n"; - } else if (Op == "SELECT LAST DELETED") { - text << "SELECT "; - size_t i = 0; - for (const auto& name : ColNames) { - text << '`' << name << '`'; - ++i; - if (i != ColNames.size()) { - text << ", "; - } - } - text << "\n FROM $todelete" << (Counter-1) << ";\n"; + } else if (Op == "SELECT LAST DELETED") { + text << "SELECT "; + size_t i = 0; + for (const auto& name : ColNames) { + text << '`' << name << '`'; + ++i; + if (i != ColNames.size()) { + text << ", "; + } + } + text << "\n FROM $todelete" << (Counter-1) << ";\n"; } else if (Op == "SELECT COUNT(*)") { text << Op ; text << "\nFROM `" << Table << "`\n"; @@ -248,57 +248,57 @@ TSqlQueryBuilder& TSqlQueryBuilder::End() text << " $count != 0;\n"; BeforeQuery << text; text.clear(); - } else { + } else { text << Op << " INTO `" << Table << "`\n"; - text << "(\n" << " "; - for (const auto& pk : Pk) { - text << "`" << pk << "`" << ","; - } + text << "(\n" << " "; + for (const auto& pk : Pk) { + text << "`" << pk << "`" << ","; + } size_t i = 0; - for (const auto& t : Fields) { - auto field = std::get<0>(t); // structured capture don't work on win32 - text << "`" << field << "`"; - i++; + for (const auto& t : Fields) { + auto field = std::get<0>(t); // structured capture don't work on win32 + text << "`" << field << "`"; + i++; if (i != Fields.size()) { - text << ","; - } - } - text << "\n)\n"; - text << "VALUES\n"; - text << "(\n" << " "; - for (const auto& pk : Pk) { - text << "$" << pk << ", "; - } - i = 0; - for (const auto& t : Fields) { - auto field = std::get<1>(t); // structured capture don't work on win32 - text << "$" << field; - i++; + text << ","; + } + } + text << "\n)\n"; + text << "VALUES\n"; + text << "(\n" << " "; + for (const auto& pk : Pk) { + text << "$" << pk << ", "; + } + i = 0; + for (const auto& t : Fields) { + auto field = std::get<1>(t); // structured capture don't work on win32 + text << "$" << field; + i++; if (i != Fields.size()) { - text << ", "; - } - } - text << "\n);\n"; - } - Fields.clear(); - AddText(text); - Counter ++; - return *this; -} - -TSqlQueryBuilder::TResult TSqlQueryBuilder::Build() -{ - TStringBuilder declr; - declr << "--!syntax_v1\n"; + text << ", "; + } + } + text << "\n);\n"; + } + Fields.clear(); + AddText(text); + Counter ++; + return *this; +} + +TSqlQueryBuilder::TResult TSqlQueryBuilder::Build() +{ + TStringBuilder declr; + declr << "--!syntax_v1\n"; declr << "-- Query name: " << QueryName << "\n"; declr << "PRAGMA TablePathPrefix(\"" << TablePrefix << "\");\n"; - for (const auto& [k, v] : ParametersOrdered) { - declr << "DECLARE $" << k << " as " << v << ";\n"; - } - declr << "\n\n"; - declr << BeforeQuery; - declr << Text; - return {declr, ParamsBuilder.Build()}; -} - + for (const auto& [k, v] : ParametersOrdered) { + declr << "DECLARE $" << k << " as " << v << ";\n"; + } + declr << "\n\n"; + declr << BeforeQuery; + declr << Text; + return {declr, ParamsBuilder.Build()}; +} + } // namespace NYq diff --git a/ydb/core/yq/libs/db_schema/db_schema.h b/ydb/core/yq/libs/db_schema/db_schema.h index 795f0f4cf3..c14a7d21c0 100644 --- a/ydb/core/yq/libs/db_schema/db_schema.h +++ b/ydb/core/yq/libs/db_schema/db_schema.h @@ -3,26 +3,26 @@ #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> -#include <util/string/builder.h> -#include <util/generic/hash_set.h> - +#include <util/string/builder.h> +#include <util/generic/hash_set.h> + namespace NYq { -class TSqlQueryBuilder { -public: +class TSqlQueryBuilder { +public: TSqlQueryBuilder(const TString& tablePrefix, const TString& queryName = "Unknown query name"); - - TSqlQueryBuilder& PushPk(); - TSqlQueryBuilder& PopPk(); - - TSqlQueryBuilder& AddPk(const TString& name, const NYdb::TValue& value); - - TSqlQueryBuilder& AddPk(const TString& name, const TString& value) { - return AddPk(name, NYdb::TValueBuilder().String(value).Build()); - } - + + TSqlQueryBuilder& PushPk(); + TSqlQueryBuilder& PopPk(); + + TSqlQueryBuilder& AddPk(const TString& name, const NYdb::TValue& value); + + TSqlQueryBuilder& AddPk(const TString& name, const TString& value) { + return AddPk(name, NYdb::TValueBuilder().String(value).Build()); + } + TSqlQueryBuilder& AddValue(const TString& name, const NYdb::TValue& value); - + TSqlQueryBuilder& AddString(const TString& name, const TString& value) { return AddValue(name, NYdb::TValueBuilder().String(value).Build()); } @@ -55,76 +55,76 @@ public: TSqlQueryBuilder& AddValue(const TString& name, const TString& value, const TString& pred) { return AddValue(name, NYdb::TValueBuilder().String(value).Build(), pred); - } - + } + TSqlQueryBuilder& AddValue(const TString& name, ui32 value, const TString& pred) { return AddValue(name, NYdb::TValueBuilder().Uint32(value).Build(), pred); - } - + } + TSqlQueryBuilder& AddValue(const TString& name, ui64 value, const TString& pred) { return AddValue(name, NYdb::TValueBuilder().Uint64(value).Build(), pred); - } - + } + TSqlQueryBuilder& AddValue(const TString& name, TInstant value, const TString& pred) { - return AddValue(name, NYdb::TValueBuilder().Timestamp(value).Build(), pred); - } - - TSqlQueryBuilder& MaybeAddValue(const TString& name, const TMaybe<NYdb::TValue>& value); - - template<typename T> - TSqlQueryBuilder& MaybeAddValue(const TString& name, const TMaybe<T>& value) { - if (value) { - AddValue(name, *value); - } - return *this; - } - + return AddValue(name, NYdb::TValueBuilder().Timestamp(value).Build(), pred); + } + + TSqlQueryBuilder& MaybeAddValue(const TString& name, const TMaybe<NYdb::TValue>& value); + + template<typename T> + TSqlQueryBuilder& MaybeAddValue(const TString& name, const TMaybe<T>& value) { + if (value) { + AddValue(name, *value); + } + return *this; + } + TSqlQueryBuilder& AddColNames(TVector<TString>&& colNames); - TSqlQueryBuilder& AddText(const TString& sql, const TString& tableName = ""); + TSqlQueryBuilder& AddText(const TString& sql, const TString& tableName = ""); TSqlQueryBuilder& AddBeforeQuery(const TString& sql, const TString& tableName = ""); - - TSqlQueryBuilder& Upsert(const TString& tableName); + + TSqlQueryBuilder& Upsert(const TString& tableName); TSqlQueryBuilder& Update(const TString& tableName); - TSqlQueryBuilder& Insert(const TString& tableName); - TSqlQueryBuilder& Delete(const TString& tableName, int limit = 0); - TSqlQueryBuilder& Get(const TString& tableName, int limit = 0); + TSqlQueryBuilder& Insert(const TString& tableName); + TSqlQueryBuilder& Delete(const TString& tableName, int limit = 0); + TSqlQueryBuilder& Get(const TString& tableName, int limit = 0); TSqlQueryBuilder& GetCount(const TString& tableName); - TSqlQueryBuilder& GetLastDeleted(); - TSqlQueryBuilder& End(); - - struct TResult { - TString Sql; - NYdb::TParams Params; - }; - - TResult Build(); - - const TString& Prefix() const { - return TablePrefix; - } - -private: - void ConstructWhereFilter(TStringBuilder& text); - - const TString TablePrefix; + TSqlQueryBuilder& GetLastDeleted(); + TSqlQueryBuilder& End(); + + struct TResult { + TString Sql; + NYdb::TParams Params; + }; + + TResult Build(); + + const TString& Prefix() const { + return TablePrefix; + } + +private: + void ConstructWhereFilter(TStringBuilder& text); + + const TString TablePrefix; const TString QueryName; - TString Table; - int Limit = 0; - TString Op; - - TVector<TVector<TString>> PkStack; - TVector<TString> Pk; - - TVector<std::tuple<TString, TString, TString>> Fields; - TVector<std::pair<TString, TString>> ParametersOrdered; - THashMap<TString, TString> Parameters; - NYdb::TParamsBuilder ParamsBuilder; - TStringBuilder BeforeQuery; - TStringBuilder Text; - int Counter = 0; + TString Table; + int Limit = 0; + TString Op; + + TVector<TVector<TString>> PkStack; + TVector<TString> Pk; + + TVector<std::tuple<TString, TString, TString>> Fields; + TVector<std::pair<TString, TString>> ParametersOrdered; + THashMap<TString, TString> Parameters; + NYdb::TParamsBuilder ParamsBuilder; + TStringBuilder BeforeQuery; + TStringBuilder Text; + int Counter = 0; TVector<TString> ColNames; -}; - +}; + } // namespace NYq diff --git a/ydb/core/yq/libs/events/events.h b/ydb/core/yq/libs/events/events.h index 68fd0e399f..b3c70e5c5f 100644 --- a/ydb/core/yq/libs/events/events.h +++ b/ydb/core/yq/libs/events/events.h @@ -10,21 +10,21 @@ #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/lib/yq/scope.h> -#include <library/cpp/actors/core/events.h> - +#include <library/cpp/actors/core/events.h> + #include <util/digest/multi.h> namespace NYq { -using NYdb::NYq::TScope; - +using NYdb::NYq::TScope; + enum class DatabaseType { Ydb, ClickHouse, DataStreams, ObjectStorage }; - + struct TQueryResult { TVector<Ydb::ResultSet> Sets; TInstant ExpirationDeadline; @@ -33,7 +33,7 @@ struct TQueryResult { struct TEvents { // Events. - struct TEvAnalyticsBase { + struct TEvAnalyticsBase { TString AuthToken; TString UserId; bool HasPerm = false; @@ -108,7 +108,7 @@ struct TEvents { , ResultSets(resultSets) {} }; - + struct TEvDbFunctionRequest : NActors::TEventLocal<TEvDbFunctionRequest, TEventIds::EvDbFunctionRequest> { using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>; TFunction Handler; @@ -156,24 +156,24 @@ struct TEvents { struct TEvEndpointRequest : NActors::TEventLocal<TEvEndpointRequest, TEventIds::EvEndpointRequest> { THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth> DatabaseIds; // DbId, DatabaseType => database auth TString YdbMvpEndpoint; - TString MdbGateway; - TString TraceId; + TString MdbGateway; + TString TraceId; bool MdbTransformHost; - - TEvEndpointRequest( + + TEvEndpointRequest( const THashMap<std::pair<TString, DatabaseType>, TDatabaseAuth>& databaseIds, - const TString& ydbMvpEndpoint, - const TString& mdbGateway, - const TString& traceId, + const TString& ydbMvpEndpoint, + const TString& mdbGateway, + const TString& traceId, bool mdbTransformHost) - : DatabaseIds(databaseIds) + : DatabaseIds(databaseIds) , YdbMvpEndpoint(ydbMvpEndpoint) - , MdbGateway(mdbGateway) - , TraceId(traceId) + , MdbGateway(mdbGateway) + , TraceId(traceId) , MdbTransformHost(mdbTransformHost) - { } - }; - + { } + }; + struct TEvDataStreamsReadRulesCreationResult : NActors::TEventLocal<TEvDataStreamsReadRulesCreationResult, TEventIds::EvDataStreamsReadRulesCreationResult> { explicit TEvDataStreamsReadRulesCreationResult(NYql::TIssues issues) : Issues(std::move(issues)) @@ -200,16 +200,16 @@ struct TEvents { YandexQuery::QueryAction Action; }; - + struct TEvForwardPingRequest : NActors::TEventLocal<TEvForwardPingRequest, TEventIds::EvForwardPingRequest> { explicit TEvForwardPingRequest(const Yq::Private::PingTaskRequest& request, bool final = false) - : Request(request) + : Request(request) , Final(final) - { } - + { } + Yq::Private::PingTaskRequest Request; bool Final; // Is this the last ping request. - }; + }; struct TEvForwardPingResponse : NActors::TEventLocal<TEvForwardPingResponse, TEventIds::EvForwardPingResponse> { TEvForwardPingResponse(bool success, YandexQuery::QueryAction action) diff --git a/ydb/core/yq/libs/gateway/empty_gateway.cpp b/ydb/core/yq/libs/gateway/empty_gateway.cpp index a763eeeb05..01bfd9bfcb 100644 --- a/ydb/core/yq/libs/gateway/empty_gateway.cpp +++ b/ydb/core/yq/libs/gateway/empty_gateway.cpp @@ -2,7 +2,7 @@ #include <ydb/core/yq/libs/graph_params/proto/graph_params.pb.h> #include <ydb/core/yq/libs/events/events.h> -#include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> +#include <ydb/core/yq/libs/tasks_packer/tasks_packer.h> #include <library/cpp/actors/core/actor.h> @@ -42,14 +42,14 @@ public: Y_UNUSED(queryParams); NProto::TGraphParams params; - THashMap<i64, TString> stagePrograms; - NTasksPacker::Pack(plan.GetTasks(), stagePrograms); + THashMap<i64, TString> stagePrograms; + NTasksPacker::Pack(plan.GetTasks(), stagePrograms); for (auto&& task : plan.GetTasks()) { *params.AddTasks() = std::move(task); } - for (const auto& [id, program]: stagePrograms) { - (*params.MutableStageProgram())[id] = program; - } + for (const auto& [id, program]: stagePrograms) { + (*params.MutableStageProgram())[id] = program; + } for (auto&& col : columns) { *params.AddColumns() = col; } diff --git a/ydb/core/yq/libs/gateway/ya.make b/ydb/core/yq/libs/gateway/ya.make index 250a5f4448..503e9b691b 100644 --- a/ydb/core/yq/libs/gateway/ya.make +++ b/ydb/core/yq/libs/gateway/ya.make @@ -13,7 +13,7 @@ PEERDIR( ydb/core/yq/libs/events ydb/core/yq/libs/read_rule ydb/core/yq/libs/shared_resources - ydb/core/yq/libs/tasks_packer + ydb/core/yq/libs/tasks_packer ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/public/issue ydb/public/api/protos diff --git a/ydb/core/yq/libs/graph_params/proto/graph_params.proto b/ydb/core/yq/libs/graph_params/proto/graph_params.proto index 4e0815c483..35291c3cf8 100644 --- a/ydb/core/yq/libs/graph_params/proto/graph_params.proto +++ b/ydb/core/yq/libs/graph_params/proto/graph_params.proto @@ -9,7 +9,7 @@ package NYq.NProto; message TGraphParams { string GraphId = 1; repeated NYql.NDqProto.TDqTask Tasks = 2; - map<int64, bytes> StageProgram = 12; + map<int64, bytes> StageProgram = 12; uint64 SourceId = 3; bytes ResultType = 4; repeated string Columns = 5; diff --git a/ydb/core/yq/libs/init/init.cpp b/ydb/core/yq/libs/init/init.cpp index 4913231539..d66ac202bf 100644 --- a/ydb/core/yq/libs/init/init.cpp +++ b/ydb/core/yq/libs/init/init.cpp @@ -1,8 +1,8 @@ -#include "init.h" - +#include "init.h" + #include <ydb/core/yq/libs/control_plane_storage/control_plane_storage.h> #include <ydb/core/yq/libs/test_connection/test_connection.h> - + #include <ydb/core/yq/libs/audit/yq_audit_service.h> #include <ydb/core/yq/libs/common/service_counters.h> #include <ydb/core/yq/libs/control_plane_proxy/control_plane_proxy.h> @@ -10,10 +10,10 @@ #include <ydb/core/yq/libs/checkpoint_storage/storage_service.h> #include <ydb/library/folder_service/folder_service.h> -#include <library/cpp/actors/http/http_proxy.h> +#include <library/cpp/actors/http/http_proxy.h> #include <library/cpp/protobuf/json/json2proto.h> #include <library/cpp/protobuf/json/proto2json.h> - + #include <ydb/library/yql/dq/actors/compute/dq_checkpoints.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/dq/comp_nodes/yql_common_dq_factory.h> @@ -36,7 +36,7 @@ #include <util/stream/file.h> #include <util/system/hostname.h> - + namespace { std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = nullptr) { @@ -93,13 +93,13 @@ std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname = n namespace NYq { -using namespace NKikimr; - -void Init( +using namespace NKikimr; + +void Init( const NYq::NConfig::TConfig& protoConfig, - ui32 nodeId, - const TActorRegistrator& actorRegistrator, - const TAppData* appData, + ui32 nodeId, + const TActorRegistrator& actorRegistrator, + const TAppData* appData, const TString& tenant, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const IYqSharedResources::TPtr& iyqSharedResources, @@ -107,8 +107,8 @@ void Init( const std::function<IActor*(const NYq::NConfig::TAuditConfig& auditConfig)>& auditServiceFactory, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const ui32& icPort - ) -{ + ) +{ Y_VERIFY(iyqSharedResources, "No YQ shared resources created"); TYqSharedResources::TPtr yqSharedResources = TYqSharedResources::Cast(iyqSharedResources); const auto clientCounters = appData->Counters->GetSubgroup("counters", "yq")->GetSubgroup("subsystem", "ClientMetrics"); @@ -158,16 +158,16 @@ void Init( auto workerManagerCounters = NYql::NDqs::TWorkerManagerCounters(yqCounters->GetSubgroup("subsystem", "worker_manager")); NKikimr::NMiniKQL::TComputationNodeFactory dqCompFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory({ - NYql::GetCommonDqFactory(), + NYql::GetCommonDqFactory(), NYql::GetDqYdbFactory(yqSharedResources->YdbDriver), - NKikimr::NMiniKQL::GetYqlFactory() - }); - + NKikimr::NMiniKQL::GetYqlFactory() + }); + NYql::TTaskTransformFactory dqTaskTransformFactory = NYql::CreateCompositeTaskTransformFactory({ - NYql::CreateCommonDqTaskTransformFactory(), - NYql::CreateYdbDqTaskTransformFactory() - }); - + NYql::CreateCommonDqTaskTransformFactory(), + NYql::CreateYdbDqTaskTransformFactory() + }); + auto sourceActorFactory = MakeIntrusive<NYql::NDq::TDqSourceFactory>(); auto sinkActorFactory = MakeIntrusive<NYql::NDq::TDqSinkFactory>(); @@ -187,7 +187,7 @@ void Init( RegisterDqPqWriteActorFactory(*sinkActorFactory, yqSharedResources->YdbDriver, credentialsFactory); RegisterDQSolomonWriteActorFactory(*sinkActorFactory, credentialsFactory); } - + ui64 mkqlInitialMemoryLimit = 8_GB; if (protoConfig.GetResourceManager().GetEnabled()) { @@ -200,22 +200,22 @@ void Init( if (!mkqlAllocSize) { mkqlAllocSize = 30_MB; } - NYql::NDqs::TLocalWorkerManagerOptions lwmOptions; - lwmOptions.Counters = workerManagerCounters; - lwmOptions.Factory = NYql::NTaskRunnerProxy::CreateFactory(appData->FunctionRegistry, dqCompFactory, dqTaskTransformFactory, false); - lwmOptions.SourceActorFactory = sourceActorFactory; - lwmOptions.SinkActorFactory = sinkActorFactory; - lwmOptions.TaskRunnerInvokerFactory = new NYql::NDqs::TTaskRunnerInvokerFactory(); - lwmOptions.MkqlInitialMemoryLimit = mkqlInitialMemoryLimit; - lwmOptions.MkqlTotalMemoryLimit = mkqlTotalMemoryLimit; - lwmOptions.MkqlMinAllocSize = mkqlAllocSize; + NYql::NDqs::TLocalWorkerManagerOptions lwmOptions; + lwmOptions.Counters = workerManagerCounters; + lwmOptions.Factory = NYql::NTaskRunnerProxy::CreateFactory(appData->FunctionRegistry, dqCompFactory, dqTaskTransformFactory, false); + lwmOptions.SourceActorFactory = sourceActorFactory; + lwmOptions.SinkActorFactory = sinkActorFactory; + lwmOptions.TaskRunnerInvokerFactory = new NYql::NDqs::TTaskRunnerInvokerFactory(); + lwmOptions.MkqlInitialMemoryLimit = mkqlInitialMemoryLimit; + lwmOptions.MkqlTotalMemoryLimit = mkqlTotalMemoryLimit; + lwmOptions.MkqlMinAllocSize = mkqlAllocSize; auto resman = NYql::NDqs::CreateLocalWorkerManager(lwmOptions); - + actorRegistrator(NYql::NDqs::MakeWorkerManagerActorID(nodeId), resman); } - + ::NYq::NCommon::TServiceCounters serviceCounters(appData->Counters); - + if (protoConfig.GetNodesManager().GetEnabled()) { const auto localAddr = GetLocalAddress(&HostName()); auto nodesManager = CreateYqlNodesManager( @@ -230,13 +230,13 @@ void Init( tenant, mkqlInitialMemoryLimit, clientCounters); - + actorRegistrator(MakeYqlNodesManagerId(), nodesManager); } - - auto httpProxy = NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance()); + + auto httpProxy = NHttp::CreateHttpProxy(*NMonitoring::TMetricRegistry::Instance()); actorRegistrator(MakeYqlAnalyticsHttpProxyId(), httpProxy); - + if (protoConfig.GetPendingFetcher().GetEnabled()) { auto fetcher = CreatePendingFetcher( yqSharedResources, @@ -255,7 +255,7 @@ void Init( std::move(pqCmConnections), clientCounters ); - + actorRegistrator(MakeYqlAnalyticsFetcherId(nodeId), fetcher); } @@ -268,8 +268,8 @@ void Init( actorRegistrator(MakeYqPrivateProxyId(), proxyPrivate); } -} - +} + IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, diff --git a/ydb/core/yq/libs/init/init.h b/ydb/core/yq/libs/init/init.h index bc2513f3b0..fa26651f18 100644 --- a/ydb/core/yq/libs/init/init.h +++ b/ydb/core/yq/libs/init/init.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include <ydb/core/base/appdata.h> - + #include <ydb/core/yq/libs/actors/nodes_manager.h> #include <ydb/core/yq/libs/actors/proxy.h> #include <ydb/core/yq/libs/actors/proxy_private.h> #include <ydb/core/yq/libs/events/events.h> #include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h> - + #include <ydb/library/folder_service/proto/config.pb.h> #include <ydb/core/yq/libs/config/protos/audit.pb.h> #include <ydb/library/yql/providers/pq/cm_client/interface/client.h> -#include <library/cpp/actors/core/actor.h> - +#include <library/cpp/actors/core/actor.h> + #include <util/generic/ptr.h> - + namespace NYq { -using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor*)>; - +using TActorRegistrator = std::function<void(NActors::TActorId, NActors::IActor*)>; + IYqSharedResources::TPtr CreateYqSharedResources( const NYq::NConfig::TConfig& config, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); -void Init( +void Init( const NYq::NConfig::TConfig& config, - ui32 nodeId, - const TActorRegistrator& actorRegistrator, - const NKikimr::TAppData* appData, + ui32 nodeId, + const TActorRegistrator& actorRegistrator, + const NKikimr::TAppData* appData, const TString& tenant, ::NPq::NConfigurationManager::IConnections::TPtr pqCmConnections, const IYqSharedResources::TPtr& yqSharedResources, diff --git a/ydb/core/yq/libs/init/ya.make b/ydb/core/yq/libs/init/ya.make index 8f408e4315..129103b024 100644 --- a/ydb/core/yq/libs/init/ya.make +++ b/ydb/core/yq/libs/init/ya.make @@ -1,12 +1,12 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - init.cpp -) - -PEERDIR( + +LIBRARY() + +SRCS( + init.cpp +) + +PEERDIR( library/cpp/actors/core library/cpp/actors/http ydb/core/base @@ -49,8 +49,8 @@ PEERDIR( ydb/library/yql/providers/ydb/actors ydb/library/yql/providers/ydb/comp_nodes ydb/library/yql/providers/ydb/provider -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/logs/log.cpp b/ydb/core/yq/libs/logs/log.cpp index c5740cff90..91c4af73ca 100644 --- a/ydb/core/yq/libs/logs/log.cpp +++ b/ydb/core/yq/libs/logs/log.cpp @@ -1,115 +1,115 @@ -#include "log.h" - +#include "log.h" + #include <ydb/core/cms/console/console.h> #include <ydb/core/cms/console/configs_dispatcher.h> - + #include <ydb/library/yql/utils/actor_log/log.h> - -#include <library/cpp/actors/core/log.h> - -namespace NKikimr { - -using namespace NActors; - -class TYqlLogsUpdater : public TActorBootstrapped<TYqlLogsUpdater> { -public: + +#include <library/cpp/actors/core/log.h> + +namespace NKikimr { + +using namespace NActors; + +class TYqlLogsUpdater : public TActorBootstrapped<TYqlLogsUpdater> { +public: static constexpr char ActorName[] = "YQ_LOGS_UPDATER"; - TYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig) - : YqlLoggerScope(new NYql::NLog::TTlsLogBackend(new TNullLogBackend())) - , LogConfig(logConfig) - { } - - void Bootstrap(const TActorContext& ctx) { - UpdateYqlLogLevels(ctx); - - // Subscribe for Logger config changes - ui32 logConfigKind = (ui32)NKikimrConsole::TConfigItem::LogConfigItem; - ctx.Send(NConsole::MakeConfigsDispatcherID(ctx.SelfID.NodeId()), - new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest( - {logConfigKind}), - IEventHandle::FlagTrackDelivery); - - Become(&TYqlLogsUpdater::MainState); - } - -private: - STFUNC(MainState) { - switch (ev->GetTypeRewrite()) { - HFunc(TEvents::TEvUndelivered, Handle); - HFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle); - HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); - default: - Y_FAIL("TYqlLogsUpdater: unexpected event type: %" PRIx32 " event: %s", - ev->GetTypeRewrite(), ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?"); - } - } - - void Handle(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { - switch (ev->Get()->SourceType) { - case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - LOG_CRIT_S(ctx, NKikimrServices::YQL_PROXY, - "Failed to deliver subscription request to config dispatcher."); - break; - - case NConsole::TEvConsole::EvConfigNotificationResponse: - LOG_ERROR_S(ctx, NKikimrServices::YQL_PROXY, "Failed to deliver config notification response."); - break; - - default: - LOG_ERROR_S(ctx, NKikimrServices::YQL_PROXY, - "Undelivered event with unexpected source type: " << ev->Get()->SourceType); - break; - } - } - - void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr &ev, const TActorContext &ctx) { - Y_UNUSED(ev); - - LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Subscribed for config changes."); - } - - void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { - auto &event = ev->Get()->Record; - - LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated table service config."); - - LogConfig.Swap(event.MutableConfig()->MutableLogConfig()); - UpdateYqlLogLevels(ctx); - } - - void UpdateYqlLogLevels(const TActorContext& ctx) { - const auto& kqpYqlName = NKikimrServices::EServiceKikimr_Name(NKikimrServices::KQP_YQL); - for (auto &entry : LogConfig.GetEntry()) { - if (entry.GetComponent() == kqpYqlName && entry.HasLevel()) { - auto yqlPriority = static_cast<NActors::NLog::EPriority>(entry.GetLevel()); - NYql::NDq::SetYqlLogLevels(yqlPriority); - LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated YQL logs priority: " - << (ui32)yqlPriority); - return; - } - } - - // Set log level based on current logger settings - ui8 currentLevel = ctx.LoggerSettings()->GetComponentSettings(NKikimrServices::KQP_YQL).Raw.X.Level; - auto yqlPriority = static_cast<NActors::NLog::EPriority>(currentLevel); - - LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated YQL logs priority to current level: " - << (ui32)yqlPriority); - NYql::NDq::SetYqlLogLevels(yqlPriority); - } - - NYql::NLog::YqlLoggerScope YqlLoggerScope; - NKikimrConfig::TLogConfig LogConfig; -}; - -NActors::IActor* CreateYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig) { - return new TYqlLogsUpdater(logConfig); -} - -TActorId MakeYqlLogsUpdaterId() { - constexpr TStringBuf name = "YQLLOGSUPD"; - return NActors::TActorId(0, name); -} - -} /* namespace NKikimr */ + TYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig) + : YqlLoggerScope(new NYql::NLog::TTlsLogBackend(new TNullLogBackend())) + , LogConfig(logConfig) + { } + + void Bootstrap(const TActorContext& ctx) { + UpdateYqlLogLevels(ctx); + + // Subscribe for Logger config changes + ui32 logConfigKind = (ui32)NKikimrConsole::TConfigItem::LogConfigItem; + ctx.Send(NConsole::MakeConfigsDispatcherID(ctx.SelfID.NodeId()), + new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest( + {logConfigKind}), + IEventHandle::FlagTrackDelivery); + + Become(&TYqlLogsUpdater::MainState); + } + +private: + STFUNC(MainState) { + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvUndelivered, Handle); + HFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle); + HFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); + default: + Y_FAIL("TYqlLogsUpdater: unexpected event type: %" PRIx32 " event: %s", + ev->GetTypeRewrite(), ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?"); + } + } + + void Handle(TEvents::TEvUndelivered::TPtr &ev, const TActorContext &ctx) { + switch (ev->Get()->SourceType) { + case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: + LOG_CRIT_S(ctx, NKikimrServices::YQL_PROXY, + "Failed to deliver subscription request to config dispatcher."); + break; + + case NConsole::TEvConsole::EvConfigNotificationResponse: + LOG_ERROR_S(ctx, NKikimrServices::YQL_PROXY, "Failed to deliver config notification response."); + break; + + default: + LOG_ERROR_S(ctx, NKikimrServices::YQL_PROXY, + "Undelivered event with unexpected source type: " << ev->Get()->SourceType); + break; + } + } + + void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr &ev, const TActorContext &ctx) { + Y_UNUSED(ev); + + LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Subscribed for config changes."); + } + + void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr &ev, const TActorContext &ctx) { + auto &event = ev->Get()->Record; + + LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated table service config."); + + LogConfig.Swap(event.MutableConfig()->MutableLogConfig()); + UpdateYqlLogLevels(ctx); + } + + void UpdateYqlLogLevels(const TActorContext& ctx) { + const auto& kqpYqlName = NKikimrServices::EServiceKikimr_Name(NKikimrServices::KQP_YQL); + for (auto &entry : LogConfig.GetEntry()) { + if (entry.GetComponent() == kqpYqlName && entry.HasLevel()) { + auto yqlPriority = static_cast<NActors::NLog::EPriority>(entry.GetLevel()); + NYql::NDq::SetYqlLogLevels(yqlPriority); + LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated YQL logs priority: " + << (ui32)yqlPriority); + return; + } + } + + // Set log level based on current logger settings + ui8 currentLevel = ctx.LoggerSettings()->GetComponentSettings(NKikimrServices::KQP_YQL).Raw.X.Level; + auto yqlPriority = static_cast<NActors::NLog::EPriority>(currentLevel); + + LOG_DEBUG_S(ctx, NKikimrServices::YQL_PROXY, "Updated YQL logs priority to current level: " + << (ui32)yqlPriority); + NYql::NDq::SetYqlLogLevels(yqlPriority); + } + + NYql::NLog::YqlLoggerScope YqlLoggerScope; + NKikimrConfig::TLogConfig LogConfig; +}; + +NActors::IActor* CreateYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig) { + return new TYqlLogsUpdater(logConfig); +} + +TActorId MakeYqlLogsUpdaterId() { + constexpr TStringBuf name = "YQLLOGSUPD"; + return NActors::TActorId(0, name); +} + +} /* namespace NKikimr */ diff --git a/ydb/core/yq/libs/logs/log.h b/ydb/core/yq/libs/logs/log.h index 668d4e12a3..82b4ec8699 100644 --- a/ydb/core/yq/libs/logs/log.h +++ b/ydb/core/yq/libs/logs/log.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/config.pb.h> - -#include <library/cpp/actors/core/actor.h> - -namespace NKikimr { - -NActors::IActor* CreateYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig); - -NActors::TActorId MakeYqlLogsUpdaterId(); - -} /* namespace NKikimr */ + +#include <library/cpp/actors/core/actor.h> + +namespace NKikimr { + +NActors::IActor* CreateYqlLogsUpdater(const NKikimrConfig::TLogConfig& logConfig); + +NActors::TActorId MakeYqlLogsUpdaterId(); + +} /* namespace NKikimr */ diff --git a/ydb/core/yq/libs/logs/ya.make b/ydb/core/yq/libs/logs/ya.make index 90f4a0e778..e56de0f1f2 100644 --- a/ydb/core/yq/libs/logs/ya.make +++ b/ydb/core/yq/libs/logs/ya.make @@ -1,17 +1,17 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - log.cpp -) - -PEERDIR( - library/cpp/actors/core + +LIBRARY() + +SRCS( + log.cpp +) + +PEERDIR( + library/cpp/actors/core ydb/core/protos ydb/library/yql/utils/actor_log -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/mock/ya.make b/ydb/core/yq/libs/mock/ya.make index 4514ea9209..37712832f6 100644 --- a/ydb/core/yq/libs/mock/ya.make +++ b/ydb/core/yq/libs/mock/ya.make @@ -1,13 +1,13 @@ OWNER(g:yq) - -LIBRARY() - -SRCS( - yql_mock.cpp -) - -PEERDIR( - library/cpp/actors/core + +LIBRARY() + +SRCS( + yql_mock.cpp +) + +PEERDIR( + library/cpp/actors/core library/cpp/json/yson library/cpp/monlib/dynamic_counters library/cpp/random_provider @@ -40,8 +40,8 @@ PEERDIR( ydb/library/yql/providers/dq/provider ydb/library/yql/providers/dq/worker_manager/interface ydb/library/yql/providers/ydb/provider -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/mock/yql_mock.cpp b/ydb/core/yq/libs/mock/yql_mock.cpp index 053d256f70..8fe694f35b 100644 --- a/ydb/core/yq/libs/mock/yql_mock.cpp +++ b/ydb/core/yq/libs/mock/yql_mock.cpp @@ -1,97 +1,97 @@ -#include "yql_mock.h" - +#include "yql_mock.h" + #include <ydb/core/testlib/actors/test_runtime.h> #include <ydb/core/yq/libs/actors/proxy.h> - + #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/http/http.h> -#include <library/cpp/actors/http/http_proxy.h> -#include <library/cpp/json/json_writer.h> -#include <library/cpp/protobuf/json/proto2json.h> -#include <library/cpp/protobuf/json/json2proto.h> - + +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/http/http.h> +#include <library/cpp/actors/http/http_proxy.h> +#include <library/cpp/json/json_writer.h> +#include <library/cpp/protobuf/json/proto2json.h> +#include <library/cpp/protobuf/json/json2proto.h> + namespace NYq { - -using namespace NActors; -using namespace NThreading; - -struct TMockLocation { - TString Endpoint = "unconfigured_endpoint"; - TString Database = "unconfigured_database"; -}; - -// actor used in tests for databaseId -> endpoint resolution -class TYqlMockActor : public NActors::TActor<TYqlMockActor> { -public: - using TBase = NActors::TActor<TYqlMockActor>; - - TYqlMockActor(int grpcPort) - : TBase(&TYqlMockActor::Handler) - { - Location.Database = "Root"; - Location.Endpoint = TStringBuilder() << "localhost:" << grpcPort; - } - + +using namespace NActors; +using namespace NThreading; + +struct TMockLocation { + TString Endpoint = "unconfigured_endpoint"; + TString Database = "unconfigured_database"; +}; + +// actor used in tests for databaseId -> endpoint resolution +class TYqlMockActor : public NActors::TActor<TYqlMockActor> { +public: + using TBase = NActors::TActor<TYqlMockActor>; + + TYqlMockActor(int grpcPort) + : TBase(&TYqlMockActor::Handler) + { + Location.Database = "Root"; + Location.Endpoint = TStringBuilder() << "localhost:" << grpcPort; + } + static constexpr char ActorName[] = "YQL_MOCK"; -private: - STRICT_STFUNC(Handler, { - HFunc(NHttp::TEvHttpProxy::TEvHttpIncomingRequest, Handle); - }); - - void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingRequest::TPtr event, - const NActors::TActorContext& ctx) - { - Y_UNUSED(ctx); - auto request = event->Get()->Request; - auto parameters = NHttp::TUrlParameters(request->URL); - TString databaseId = parameters["databaseId"]; - - NJson::TJsonValue json; - bool ok = true; - - if (databaseId.Contains("FakeDatabaseId")) { - json["endpoint"] = Location.Endpoint + "/?database=" + Location.Database; - } else { - ok = false; - json["message"] = "Database not found"; - } - - TStringStream stream; - NJson::WriteJson(&stream, &json); - NHttp::THttpOutgoingResponsePtr response = (ok) - ? request->CreateResponseOK(stream.Str(), "application/json; charset=utf-8") - : request->CreateResponseNotFound(stream.Str(), "application/json; charset=utf-8"); - Send(event->Sender, new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(response)); - } - - TMockLocation Location; -}; - +private: + STRICT_STFUNC(Handler, { + HFunc(NHttp::TEvHttpProxy::TEvHttpIncomingRequest, Handle); + }); + + void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingRequest::TPtr event, + const NActors::TActorContext& ctx) + { + Y_UNUSED(ctx); + auto request = event->Get()->Request; + auto parameters = NHttp::TUrlParameters(request->URL); + TString databaseId = parameters["databaseId"]; + + NJson::TJsonValue json; + bool ok = true; + + if (databaseId.Contains("FakeDatabaseId")) { + json["endpoint"] = Location.Endpoint + "/?database=" + Location.Database; + } else { + ok = false; + json["message"] = "Database not found"; + } + + TStringStream stream; + NJson::WriteJson(&stream, &json); + NHttp::THttpOutgoingResponsePtr response = (ok) + ? request->CreateResponseOK(stream.Str(), "application/json; charset=utf-8") + : request->CreateResponseNotFound(stream.Str(), "application/json; charset=utf-8"); + Send(event->Sender, new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(response)); + } + + TMockLocation Location; +}; + void InitTest(NActors::TTestActorRuntime* runtime, int httpPort, int grpcPort, const IYqSharedResources::TPtr& yqSharedResources) -{ +{ yqSharedResources->Init(runtime->GetAnyNodeActorSystem()); auto httpProxyId = NYq::MakeYqlAnalyticsHttpProxyId(); - - TActorId mockActorId = runtime->Register(CreateYqlMockActor(grpcPort)); - - runtime->Send(new NActors::IEventHandle( - httpProxyId, TActorId(), - new NHttp::TEvHttpProxy::TEvAddListeningPort(httpPort)), 0, true); - - runtime->Send(new NActors::IEventHandle( - httpProxyId, TActorId(), - new NHttp::TEvHttpProxy::TEvRegisterHandler("/yql-mock/abc/database", mockActorId)), - 0, true); -} - -NActors::IActor* CreateYqlMockActor(int grpcPort) { - return new TYqlMockActor(grpcPort); -} - + + TActorId mockActorId = runtime->Register(CreateYqlMockActor(grpcPort)); + + runtime->Send(new NActors::IEventHandle( + httpProxyId, TActorId(), + new NHttp::TEvHttpProxy::TEvAddListeningPort(httpPort)), 0, true); + + runtime->Send(new NActors::IEventHandle( + httpProxyId, TActorId(), + new NHttp::TEvHttpProxy::TEvRegisterHandler("/yql-mock/abc/database", mockActorId)), + 0, true); +} + +NActors::IActor* CreateYqlMockActor(int grpcPort) { + return new TYqlMockActor(grpcPort); +} + } // namespace NYq diff --git a/ydb/core/yq/libs/mock/yql_mock.h b/ydb/core/yq/libs/mock/yql_mock.h index 3f6be749e8..da8656b0ff 100644 --- a/ydb/core/yq/libs/mock/yql_mock.h +++ b/ydb/core/yq/libs/mock/yql_mock.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <ydb/core/testlib/actors/test_runtime.h> #include <ydb/core/yq/libs/shared_resources/interface/shared_resources.h> - -#include <library/cpp/actors/core/actorsystem.h> - + +#include <library/cpp/actors/core/actorsystem.h> + namespace NYq { - -NActors::IActor* CreateYqlMockActor(int grpcPort); + +NActors::IActor* CreateYqlMockActor(int grpcPort); void InitTest(NActors::TTestActorRuntime* runtime, int httpPort, int grpcPort, const IYqSharedResources::TPtr& yqSharedResources); - + } // namespace NYq diff --git a/ydb/core/yq/libs/private_client/utils.cpp b/ydb/core/yq/libs/private_client/utils.cpp index 7117fc1712..f1ba5c7e22 100644 --- a/ydb/core/yq/libs/private_client/utils.cpp +++ b/ydb/core/yq/libs/private_client/utils.cpp @@ -5,8 +5,8 @@ namespace NYq { -using namespace NYdb::NYq; - +using namespace NYdb::NYq; + void UpdateConnections( TClient& client, const TString& folderId, diff --git a/ydb/core/yq/libs/private_client/utils.h b/ydb/core/yq/libs/private_client/utils.h index 5e9a7878ca..ff1b255c31 100644 --- a/ydb/core/yq/libs/private_client/utils.h +++ b/ydb/core/yq/libs/private_client/utils.h @@ -5,8 +5,8 @@ namespace NYq { void UpdateConnections( - NYdb::NYq::TClient& client, - const TString& folderId, - const TString& connectionsStr); + NYdb::NYq::TClient& client, + const TString& folderId, + const TString& connectionsStr); } // NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.cpp b/ydb/core/yq/libs/result_formatter/result_formatter.cpp index 10a2345626..394832b7e5 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter.cpp @@ -1,5 +1,5 @@ -#include "result_formatter.h" - +#include "result_formatter.h" + #include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h> #include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h> #include <ydb/library/yql/providers/common/codec/yql_codec.h> @@ -7,334 +7,334 @@ #include <ydb/library/yql/ast/yql_expr.h> #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - + #include <ydb/core/engine/mkql_proto.h> #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> - -#include <library/cpp/json/yson/json2yson.h> - + +#include <library/cpp/json/yson/json2yson.h> + namespace NYq { - -using namespace NKikimr::NMiniKQL; + +using namespace NKikimr::NMiniKQL; using NYql::NUdf::TUnboxedValuePod; - -namespace { - -const NYql::TTypeAnnotationNode* MakePrimitiveType(NYdb::TTypeParser& parser, NYql::TExprContext& ctx) -{ - auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << parser.GetPrimitive()); - return ctx.MakeType<NYql::TDataExprType>(dataSlot); -} - -const NYql::TTypeAnnotationNode* MakeDecimalType(NYdb::TTypeParser& parser, NYql::TExprContext& ctx) -{ - auto decimal = parser.GetDecimal(); - auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")"); - return ctx.MakeType<NYql::TDataExprType>(dataSlot); -} - -NKikimr::NMiniKQL::TType* MakePrimitiveType(NYdb::TTypeParser& parser, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << parser.GetPrimitive()); - return TDataType::Create(GetDataTypeInfo(dataSlot).TypeId, env); -} - -NKikimr::NMiniKQL::TType* MakeDecimalType(NYdb::TTypeParser& parser, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - auto decimal = parser.GetDecimal(); - return TDataDecimalType::Create((ui8)decimal.Precision, (ui8)decimal.Scale, env); -} - -const NYql::TTypeAnnotationNode* MakeOptionalType(const NYql::TTypeAnnotationNode* underlying, NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TOptionalExprType>(underlying); -} - -NKikimr::NMiniKQL::TType* MakeOptionalType(NKikimr::NMiniKQL::TType* underlying, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TOptionalType::Create(underlying, env); -} - -const NYql::TTypeAnnotationNode* MakeListType(const NYql::TTypeAnnotationNode* underlying, NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TListExprType>(underlying); -} - -NKikimr::NMiniKQL::TType* MakeListType(NKikimr::NMiniKQL::TType* underlying, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TListType::Create(underlying, env); -} - -const NYql::TTypeAnnotationNode* MakeStructType( - const TVector<std::pair<TString, const NYql::TTypeAnnotationNode*>>& i, - NYql::TExprContext& ctx) -{ - TVector<const NYql::TItemExprType*> items; + +namespace { + +const NYql::TTypeAnnotationNode* MakePrimitiveType(NYdb::TTypeParser& parser, NYql::TExprContext& ctx) +{ + auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << parser.GetPrimitive()); + return ctx.MakeType<NYql::TDataExprType>(dataSlot); +} + +const NYql::TTypeAnnotationNode* MakeDecimalType(NYdb::TTypeParser& parser, NYql::TExprContext& ctx) +{ + auto decimal = parser.GetDecimal(); + auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")"); + return ctx.MakeType<NYql::TDataExprType>(dataSlot); +} + +NKikimr::NMiniKQL::TType* MakePrimitiveType(NYdb::TTypeParser& parser, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + auto dataSlot = NYql::NUdf::GetDataSlot(TStringBuilder() << parser.GetPrimitive()); + return TDataType::Create(GetDataTypeInfo(dataSlot).TypeId, env); +} + +NKikimr::NMiniKQL::TType* MakeDecimalType(NYdb::TTypeParser& parser, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + auto decimal = parser.GetDecimal(); + return TDataDecimalType::Create((ui8)decimal.Precision, (ui8)decimal.Scale, env); +} + +const NYql::TTypeAnnotationNode* MakeOptionalType(const NYql::TTypeAnnotationNode* underlying, NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TOptionalExprType>(underlying); +} + +NKikimr::NMiniKQL::TType* MakeOptionalType(NKikimr::NMiniKQL::TType* underlying, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TOptionalType::Create(underlying, env); +} + +const NYql::TTypeAnnotationNode* MakeListType(const NYql::TTypeAnnotationNode* underlying, NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TListExprType>(underlying); +} + +NKikimr::NMiniKQL::TType* MakeListType(NKikimr::NMiniKQL::TType* underlying, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TListType::Create(underlying, env); +} + +const NYql::TTypeAnnotationNode* MakeStructType( + const TVector<std::pair<TString, const NYql::TTypeAnnotationNode*>>& i, + NYql::TExprContext& ctx) +{ + TVector<const NYql::TItemExprType*> items; items.reserve(i.size()); - for (const auto& [k, v] : i) { - items.push_back(ctx.MakeType<NYql::TItemExprType>(k, v)); - } - return ctx.MakeType<NYql::TStructExprType>(items); -} - -NKikimr::NMiniKQL::TType* MakeStructType( - const TVector<std::pair<TString, NKikimr::NMiniKQL::TType*>>& items, - NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TStructType::Create(&items[0], items.size(), env); -} - -const NYql::TTypeAnnotationNode* MakeTupleType( - const TVector<const NYql::TTypeAnnotationNode*>& items, - NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TTupleExprType>(items); -} - -NKikimr::NMiniKQL::TType* MakeTupleType( - const TVector<NKikimr::NMiniKQL::TType*>& items, - NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TTupleType::Create(items.size(), &items[0], env); -} - -const NYql::TTypeAnnotationNode* MakeDictType( - const NYql::TTypeAnnotationNode* key, - const NYql::TTypeAnnotationNode* payload, - NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TDictExprType>(key, payload); -} - -NKikimr::NMiniKQL::TType* MakeDictType( - NKikimr::NMiniKQL::TType* key, - NKikimr::NMiniKQL::TType* payload, - NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TDictType::Create(key, payload, env); -} - -const NYql::TTypeAnnotationNode* MakeVoidType(NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TVoidExprType>(); -} - -NKikimr::NMiniKQL::TType* MakeVoidType(NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return env.GetTypeOfVoid(); -} - -const NYql::TTypeAnnotationNode* MakeNullType(NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TNullExprType>(); -} - -NKikimr::NMiniKQL::TType* MakeNullType(NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return env.GetTypeOfNull(); -} - -const NYql::TTypeAnnotationNode* MakeEmptyListType(NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TEmptyListExprType>(); -} - -NKikimr::NMiniKQL::TType* MakeEmptyListType(NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return env.GetTypeOfEmptyList(); -} - -const NYql::TTypeAnnotationNode* MakeEmptyDictType(NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TEmptyDictExprType>(); -} - -NKikimr::NMiniKQL::TType* MakeEmptyDictType(NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return env.GetTypeOfEmptyDict(); -} - + for (const auto& [k, v] : i) { + items.push_back(ctx.MakeType<NYql::TItemExprType>(k, v)); + } + return ctx.MakeType<NYql::TStructExprType>(items); +} + +NKikimr::NMiniKQL::TType* MakeStructType( + const TVector<std::pair<TString, NKikimr::NMiniKQL::TType*>>& items, + NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TStructType::Create(&items[0], items.size(), env); +} + +const NYql::TTypeAnnotationNode* MakeTupleType( + const TVector<const NYql::TTypeAnnotationNode*>& items, + NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TTupleExprType>(items); +} + +NKikimr::NMiniKQL::TType* MakeTupleType( + const TVector<NKikimr::NMiniKQL::TType*>& items, + NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TTupleType::Create(items.size(), &items[0], env); +} + +const NYql::TTypeAnnotationNode* MakeDictType( + const NYql::TTypeAnnotationNode* key, + const NYql::TTypeAnnotationNode* payload, + NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TDictExprType>(key, payload); +} + +NKikimr::NMiniKQL::TType* MakeDictType( + NKikimr::NMiniKQL::TType* key, + NKikimr::NMiniKQL::TType* payload, + NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TDictType::Create(key, payload, env); +} + +const NYql::TTypeAnnotationNode* MakeVoidType(NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TVoidExprType>(); +} + +NKikimr::NMiniKQL::TType* MakeVoidType(NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return env.GetTypeOfVoid(); +} + +const NYql::TTypeAnnotationNode* MakeNullType(NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TNullExprType>(); +} + +NKikimr::NMiniKQL::TType* MakeNullType(NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return env.GetTypeOfNull(); +} + +const NYql::TTypeAnnotationNode* MakeEmptyListType(NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TEmptyListExprType>(); +} + +NKikimr::NMiniKQL::TType* MakeEmptyListType(NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return env.GetTypeOfEmptyList(); +} + +const NYql::TTypeAnnotationNode* MakeEmptyDictType(NYql::TExprContext& ctx) +{ + return ctx.MakeType<NYql::TEmptyDictExprType>(); +} + +NKikimr::NMiniKQL::TType* MakeEmptyDictType(NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return env.GetTypeOfEmptyDict(); +} + const NYql::TTypeAnnotationNode* MakeVariantType(const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TVariantExprType>(underlyingType); -} - -NKikimr::NMiniKQL::TType* MakeVariantType(NKikimr::NMiniKQL::TType* underlyingType, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TVariantType::Create(underlyingType, env); -} - +{ + return ctx.MakeType<NYql::TVariantExprType>(underlyingType); +} + +NKikimr::NMiniKQL::TType* MakeVariantType(NKikimr::NMiniKQL::TType* underlyingType, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TVariantType::Create(underlyingType, env); +} + const NYql::TTypeAnnotationNode* MakeTaggedType(const TString& tag, const NYql::TTypeAnnotationNode* underlyingType, NYql::TExprContext& ctx) -{ - return ctx.MakeType<NYql::TTaggedExprType>(underlyingType, tag); -} - -NKikimr::NMiniKQL::TType* MakeTaggedType(const TString& tag, NKikimr::NMiniKQL::TType* underlyingType, NKikimr::NMiniKQL::TTypeEnvironment& env) -{ - return TTaggedType::Create(underlyingType, tag, env); -} - -template<typename TType, typename TContext> -TType MakeType(NYdb::TTypeParser& parser, TContext& env) -{ - switch (parser.GetKind()) { - case NYdb::TTypeParser::ETypeKind::Primitive: { - return MakePrimitiveType(parser, env); - } - case NYdb::TTypeParser::ETypeKind::Decimal: { - return MakeDecimalType(parser, env); - } - case NYdb::TTypeParser::ETypeKind::Optional: { - parser.OpenOptional(); - auto underlying = MakeType<TType>(parser, env); - if (!underlying) { - return nullptr; - } - parser.CloseOptional(); - return MakeOptionalType(underlying, env); - } - case NYdb::TTypeParser::ETypeKind::List: { - parser.OpenList(); - auto underlying = MakeType<TType>(parser, env); - auto node = MakeListType(underlying, env); - parser.CloseList(); - return node; - } - case NYdb::TTypeParser::ETypeKind::Struct: { - TVector<std::pair<TString, TType>> items; - parser.OpenStruct(); - TType node = nullptr; - while (parser.TryNextMember()) { - auto colName = parser.GetMemberName(); - node = MakeType<TType>(parser, env); - if (!node) { - break; - } - items.push_back({colName, node}); - } - parser.CloseStruct(); - if (!node) { - return nullptr; - } - - return MakeStructType(items, env); - } - case NYdb::TTypeParser::ETypeKind::Tuple: { - TVector<TType> items; - TType node = nullptr; - parser.OpenTuple(); - while (parser.TryNextElement()) { - node = MakeType<TType>(parser, env); - if (!node) { - break; - } - items.push_back(node); - } - if (!node) { - return nullptr; - } - parser.CloseTuple(); - return MakeTupleType(items, env); - } - case NYdb::TTypeParser::ETypeKind::Dict: { - parser.OpenDict(); - parser.DictKey(); - auto* key = MakeType<TType>(parser, env); - if (!key) { - return nullptr; - } - parser.DictPayload(); - auto* payload = MakeType<TType>(parser, env); - if (!payload) { - return nullptr; - } - parser.CloseDict(); - return MakeDictType(key, payload, env); - } - case NYdb::TTypeParser::ETypeKind::Variant: { - parser.OpenVariant(); - auto node = MakeVariantType(MakeType<TType>(parser, env), env); - parser.CloseVariant(); - return node; - } - case NYdb::TTypeParser::ETypeKind::Tagged: { - parser.OpenTagged(); - auto tag = parser.GetTag(); - auto node = MakeTaggedType(tag, MakeType<TType>(parser, env), env); - parser.CloseTagged(); - return node; - } - case NYdb::TTypeParser::ETypeKind::Void: { - return MakeVoidType(env); - } - case NYdb::TTypeParser::ETypeKind::Null: { - return MakeNullType(env); - } - case NYdb::TTypeParser::ETypeKind::EmptyList: { - return MakeEmptyListType(env); - } - case NYdb::TTypeParser::ETypeKind::EmptyDict: { - return MakeEmptyDictType(env); - } - default: - return nullptr; - } -} - -struct TTypePair { +{ + return ctx.MakeType<NYql::TTaggedExprType>(underlyingType, tag); +} + +NKikimr::NMiniKQL::TType* MakeTaggedType(const TString& tag, NKikimr::NMiniKQL::TType* underlyingType, NKikimr::NMiniKQL::TTypeEnvironment& env) +{ + return TTaggedType::Create(underlyingType, tag, env); +} + +template<typename TType, typename TContext> +TType MakeType(NYdb::TTypeParser& parser, TContext& env) +{ + switch (parser.GetKind()) { + case NYdb::TTypeParser::ETypeKind::Primitive: { + return MakePrimitiveType(parser, env); + } + case NYdb::TTypeParser::ETypeKind::Decimal: { + return MakeDecimalType(parser, env); + } + case NYdb::TTypeParser::ETypeKind::Optional: { + parser.OpenOptional(); + auto underlying = MakeType<TType>(parser, env); + if (!underlying) { + return nullptr; + } + parser.CloseOptional(); + return MakeOptionalType(underlying, env); + } + case NYdb::TTypeParser::ETypeKind::List: { + parser.OpenList(); + auto underlying = MakeType<TType>(parser, env); + auto node = MakeListType(underlying, env); + parser.CloseList(); + return node; + } + case NYdb::TTypeParser::ETypeKind::Struct: { + TVector<std::pair<TString, TType>> items; + parser.OpenStruct(); + TType node = nullptr; + while (parser.TryNextMember()) { + auto colName = parser.GetMemberName(); + node = MakeType<TType>(parser, env); + if (!node) { + break; + } + items.push_back({colName, node}); + } + parser.CloseStruct(); + if (!node) { + return nullptr; + } + + return MakeStructType(items, env); + } + case NYdb::TTypeParser::ETypeKind::Tuple: { + TVector<TType> items; + TType node = nullptr; + parser.OpenTuple(); + while (parser.TryNextElement()) { + node = MakeType<TType>(parser, env); + if (!node) { + break; + } + items.push_back(node); + } + if (!node) { + return nullptr; + } + parser.CloseTuple(); + return MakeTupleType(items, env); + } + case NYdb::TTypeParser::ETypeKind::Dict: { + parser.OpenDict(); + parser.DictKey(); + auto* key = MakeType<TType>(parser, env); + if (!key) { + return nullptr; + } + parser.DictPayload(); + auto* payload = MakeType<TType>(parser, env); + if (!payload) { + return nullptr; + } + parser.CloseDict(); + return MakeDictType(key, payload, env); + } + case NYdb::TTypeParser::ETypeKind::Variant: { + parser.OpenVariant(); + auto node = MakeVariantType(MakeType<TType>(parser, env), env); + parser.CloseVariant(); + return node; + } + case NYdb::TTypeParser::ETypeKind::Tagged: { + parser.OpenTagged(); + auto tag = parser.GetTag(); + auto node = MakeTaggedType(tag, MakeType<TType>(parser, env), env); + parser.CloseTagged(); + return node; + } + case NYdb::TTypeParser::ETypeKind::Void: { + return MakeVoidType(env); + } + case NYdb::TTypeParser::ETypeKind::Null: { + return MakeNullType(env); + } + case NYdb::TTypeParser::ETypeKind::EmptyList: { + return MakeEmptyListType(env); + } + case NYdb::TTypeParser::ETypeKind::EmptyDict: { + return MakeEmptyDictType(env); + } + default: + return nullptr; + } +} + +struct TTypePair { NKikimr::NMiniKQL::TType* MiniKQLType = nullptr; const NYql::TTypeAnnotationNode* TypeAnnotation = nullptr; -}; - +}; + TTypePair FormatColumnType( - NJson::TJsonValue& root, - NYdb::TType type, - NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, - NYql::TExprContext& ctx) -{ + NJson::TJsonValue& root, + NYdb::TType type, + NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, + NYql::TExprContext& ctx) +{ TTypePair result; - NYdb::TTypeParser parser(type); - result.MiniKQLType = MakeType<NKikimr::NMiniKQL::TType*>(parser, typeEnv); - result.TypeAnnotation = MakeType<const NYql::TTypeAnnotationNode*>(parser, ctx); - // TODO: use - // NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, IOutputStream& err, bool withTagged = false); - - if (!result.MiniKQLType) { - root = "Null"; - return result; - } - - //NJson::ReadJsonTree( - // NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteTypeToYson(result.MiniKQLType)), - // &root); - - NJson::ReadJsonTree( - NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteTypeToYson(result.TypeAnnotation)), - &root); - - return result; -} - -void FormatColumnValue( - NJson::TJsonValue& root, - const NYdb::TValue& value, - NKikimr::NMiniKQL::TType* type, - const THolderFactory& holderFactory) -{ - const Ydb::Value& rawProtoValue = NYdb::TProtoAccessor::GetProto(value); - + NYdb::TTypeParser parser(type); + result.MiniKQLType = MakeType<NKikimr::NMiniKQL::TType*>(parser, typeEnv); + result.TypeAnnotation = MakeType<const NYql::TTypeAnnotationNode*>(parser, ctx); + // TODO: use + // NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, IOutputStream& err, bool withTagged = false); + + if (!result.MiniKQLType) { + root = "Null"; + return result; + } + + //NJson::ReadJsonTree( + // NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteTypeToYson(result.MiniKQLType)), + // &root); + + NJson::ReadJsonTree( + NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteTypeToYson(result.TypeAnnotation)), + &root); + + return result; +} + +void FormatColumnValue( + NJson::TJsonValue& root, + const NYdb::TValue& value, + NKikimr::NMiniKQL::TType* type, + const THolderFactory& holderFactory) +{ + const Ydb::Value& rawProtoValue = NYdb::TProtoAccessor::GetProto(value); + NYql::NUdf::TUnboxedValue unboxed = ImportValueFromProto( - type, - rawProtoValue, - holderFactory); - - NJson::ReadJsonTree( - NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteYsonValue(unboxed, type)), - &root); -} - -} // namespace - + type, + rawProtoValue, + holderFactory); + + NJson::ReadJsonTree( + NJson2Yson::ConvertYson2Json(NYql::NCommon::WriteYsonValue(unboxed, type)), + &root); +} + +} // namespace + TString FormatSchema(const YandexQuery::Schema& schema) { NYql::TExprContext ctx; @@ -349,44 +349,44 @@ TString FormatSchema(const YandexQuery::Schema& schema) return NYql::NCommon::WriteTypeToYson(MakeStructType(typedColumns, ctx), NYson::EYsonFormat::Text); } -void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet) -{ - NYql::TExprContext ctx; - NKikimr::NMiniKQL::TScopedAlloc alloc; - NKikimr::NMiniKQL::TTypeEnvironment typeEnv(alloc); - - TMemoryUsageInfo memInfo("BuildYdbResultSet"); - THolderFactory holderFactory(alloc.Ref(), memInfo); - - - NJson::TJsonValue& columns = root["columns"]; - const auto& columnsMeta = resultSet.GetColumnsMeta(); - - TVector<TTypePair> columnTypes; - columnTypes.resize(columnsMeta.size()); - - int i = 0; - for (const NYdb::TColumn& columnMeta : columnsMeta) { - NJson::TJsonValue& column = columns.AppendValue(NJson::TJsonValue()); - column["name"] = columnMeta.Name; - columnTypes[i++] = FormatColumnType(column["type"], columnMeta.Type, typeEnv, ctx); - } - - NJson::TJsonValue& data = root["data"]; - data.SetType(NJson::JSON_ARRAY); - - NYdb::TResultSetParser rsParser(resultSet); - while (rsParser.TryNextRow()) { - NJson::TJsonValue& row = data.AppendValue(NJson::TJsonValue()); - for (size_t columnNum = 0; columnNum < columnsMeta.size(); ++columnNum) { - const NYdb::TColumn& columnMeta = columnsMeta[columnNum]; - FormatColumnValue( - row[columnMeta.Name], - rsParser.GetValue(columnNum), - columnTypes[columnNum].MiniKQLType, - holderFactory); - } - } -} - +void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet) +{ + NYql::TExprContext ctx; + NKikimr::NMiniKQL::TScopedAlloc alloc; + NKikimr::NMiniKQL::TTypeEnvironment typeEnv(alloc); + + TMemoryUsageInfo memInfo("BuildYdbResultSet"); + THolderFactory holderFactory(alloc.Ref(), memInfo); + + + NJson::TJsonValue& columns = root["columns"]; + const auto& columnsMeta = resultSet.GetColumnsMeta(); + + TVector<TTypePair> columnTypes; + columnTypes.resize(columnsMeta.size()); + + int i = 0; + for (const NYdb::TColumn& columnMeta : columnsMeta) { + NJson::TJsonValue& column = columns.AppendValue(NJson::TJsonValue()); + column["name"] = columnMeta.Name; + columnTypes[i++] = FormatColumnType(column["type"], columnMeta.Type, typeEnv, ctx); + } + + NJson::TJsonValue& data = root["data"]; + data.SetType(NJson::JSON_ARRAY); + + NYdb::TResultSetParser rsParser(resultSet); + while (rsParser.TryNextRow()) { + NJson::TJsonValue& row = data.AppendValue(NJson::TJsonValue()); + for (size_t columnNum = 0; columnNum < columnsMeta.size(); ++columnNum) { + const NYdb::TColumn& columnMeta = columnsMeta[columnNum]; + FormatColumnValue( + row[columnMeta.Name], + rsParser.GetValue(columnNum), + columnTypes[columnNum].MiniKQLType, + holderFactory); + } + } +} + } // namespace NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter.h b/ydb/core/yq/libs/result_formatter/result_formatter.h index 6eabe0c692..031196e050 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter.h +++ b/ydb/core/yq/libs/result_formatter/result_formatter.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <ydb/public/api/protos/yq.pb.h> #include <ydb/public/sdk/cpp/client/ydb_value/value.h> #include <ydb/public/sdk/cpp/client/ydb_result/result.h> - -#include <util/generic/string.h> - -#include <library/cpp/json/json_writer.h> - + +#include <util/generic/string.h> + +#include <library/cpp/json/json_writer.h> + namespace NYq { - -void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet); + +void FormatResultSet(NJson::TJsonValue& root, const NYdb::TResultSet& resultSet); TString FormatSchema(const YandexQuery::Schema& schema); - + } // namespace NYq diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp index 14e7e0d8ba..bebbf68ab3 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp @@ -1,398 +1,398 @@ -#include "result_formatter.h" - +#include "result_formatter.h" + #include <ydb/services/ydb/ydb_common_ut.h> - + using namespace NYq; - -Y_UNIT_TEST_SUITE(ResultFormatter) { - Y_UNIT_TEST(Primitive) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& type = *column.mutable_type(); - type.set_type_id(Ydb::Type::INT32); - } - { - auto& column = *rs.add_columns(); - column.set_name("column1"); - auto& type = *column.mutable_type(); - type.set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - value.add_items()->set_int32_value(31337); - value.add_items()->set_int64_value(1000000001); - } - { - auto& value = *rs.add_rows(); - value.add_items()->set_int32_value(31338); - value.add_items()->set_int64_value(1000000002); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - // Cerr << stream.Str() << Endl; - - TString expected = R"___({"data":[{"column0":"31337","column1":"1000000001"},{"column0":"31338","column1":"1000000002"}],"columns":[{"name":"column0","type":["DataType","Int32"]},{"name":"column1","type":["DataType","Int64"]}]})___"; - - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(List) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& type = *column.mutable_type()->mutable_list_type()->mutable_item(); - type.set_type_id(Ydb::Type::INT32); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.add_items()->set_int32_value(31337); - cell.add_items()->set_int32_value(1000000001); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - //Cerr << stream.Str() << Endl; - - TString expected = R"___({"data":[{"column0":["31337","1000000001"]}],"columns":[{"name":"column0","type":["ListType",["DataType","Int32"]]}]})___"; - - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Optional) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& type = *column.mutable_type()->mutable_optional_type()->mutable_item(); - type.set_type_id(Ydb::Type::INT32); - } - { - auto& value = *rs.add_rows(); - value.add_items()->set_int32_value(31337); - } - { - auto& value = *rs.add_rows(); - value.add_items()->set_null_flag_value(::google::protobuf::NullValue()); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - //Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":["31337"]},{"column0":null}],"columns":[{"name":"column0","type":["OptionalType",["DataType","Int32"]]}]})___"; - - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Struct) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& struct_type = *column.mutable_type()->mutable_struct_type(); - auto& m1 = *struct_type.add_members(); - auto& m2 = *struct_type.add_members(); - - m1.set_name("k1"); - m1.mutable_type()->set_type_id(Ydb::Type::INT32); - - m2.set_name("k2"); - m2.mutable_type()->set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.add_items()->set_int32_value(31337); // k1 - cell.add_items()->set_int64_value(113370); // k2 - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - //Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":["31337","113370"]}],"columns":[{"name":"column0","type":["StructType",[["k1",["DataType","Int32"]],["k2",["DataType","Int64"]]]]}]})___"; - - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Void) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - column.mutable_type()->set_void_type(google::protobuf::NullValue()); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - Y_UNUSED(cell); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - // Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":"Void"}],"columns":[{"name":"column0","type":["VoidType"]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Null) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - column.mutable_type()->set_null_type(google::protobuf::NullValue()); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - Y_UNUSED(cell); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - // Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":null}],"columns":[{"name":"column0","type":["NullType"]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(EmptyList) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - column.mutable_type()->set_empty_list_type(google::protobuf::NullValue()); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - Y_UNUSED(cell); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - // Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":[]}],"columns":[{"name":"column0","type":["EmptyListType"]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(EmptyDict) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - column.mutable_type()->set_empty_dict_type(google::protobuf::NullValue()); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - Y_UNUSED(cell); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - // Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":[]}],"columns":[{"name":"column0","type":["EmptyDictType"]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Tuple) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& tuple_type = *column.mutable_type()->mutable_tuple_type(); - tuple_type.add_elements()->set_type_id(Ydb::Type::INT32); - tuple_type.add_elements()->set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.add_items()->set_int32_value(31337); // k1 - cell.add_items()->set_int64_value(113370); // k2 - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - //Cerr << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":["31337","113370"]}],"columns":[{"name":"column0","type":["TupleType",[["DataType","Int32"],["DataType","Int64"]]]}]})___"; - - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Dict) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& dict_type = *column.mutable_type()->mutable_dict_type(); - dict_type.mutable_key()->set_type_id(Ydb::Type::INT32); - dict_type.mutable_payload()->set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - auto* pair = cell.add_pairs(); - pair->mutable_key()->set_int32_value(31337); - pair->mutable_payload()->set_int64_value(113370); - - pair = cell.add_pairs(); - pair->mutable_key()->set_int32_value(113370); - pair->mutable_payload()->set_int64_value(31337); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - - //Cerr << stream.Str() << Endl; + +Y_UNIT_TEST_SUITE(ResultFormatter) { + Y_UNIT_TEST(Primitive) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& type = *column.mutable_type(); + type.set_type_id(Ydb::Type::INT32); + } + { + auto& column = *rs.add_columns(); + column.set_name("column1"); + auto& type = *column.mutable_type(); + type.set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + value.add_items()->set_int32_value(31337); + value.add_items()->set_int64_value(1000000001); + } + { + auto& value = *rs.add_rows(); + value.add_items()->set_int32_value(31338); + value.add_items()->set_int64_value(1000000002); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + // Cerr << stream.Str() << Endl; + + TString expected = R"___({"data":[{"column0":"31337","column1":"1000000001"},{"column0":"31338","column1":"1000000002"}],"columns":[{"name":"column0","type":["DataType","Int32"]},{"name":"column1","type":["DataType","Int64"]}]})___"; + + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(List) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& type = *column.mutable_type()->mutable_list_type()->mutable_item(); + type.set_type_id(Ydb::Type::INT32); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.add_items()->set_int32_value(31337); + cell.add_items()->set_int32_value(1000000001); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + //Cerr << stream.Str() << Endl; + + TString expected = R"___({"data":[{"column0":["31337","1000000001"]}],"columns":[{"name":"column0","type":["ListType",["DataType","Int32"]]}]})___"; + + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Optional) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& type = *column.mutable_type()->mutable_optional_type()->mutable_item(); + type.set_type_id(Ydb::Type::INT32); + } + { + auto& value = *rs.add_rows(); + value.add_items()->set_int32_value(31337); + } + { + auto& value = *rs.add_rows(); + value.add_items()->set_null_flag_value(::google::protobuf::NullValue()); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + //Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":["31337"]},{"column0":null}],"columns":[{"name":"column0","type":["OptionalType",["DataType","Int32"]]}]})___"; + + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Struct) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& struct_type = *column.mutable_type()->mutable_struct_type(); + auto& m1 = *struct_type.add_members(); + auto& m2 = *struct_type.add_members(); + + m1.set_name("k1"); + m1.mutable_type()->set_type_id(Ydb::Type::INT32); + + m2.set_name("k2"); + m2.mutable_type()->set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.add_items()->set_int32_value(31337); // k1 + cell.add_items()->set_int64_value(113370); // k2 + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + //Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":["31337","113370"]}],"columns":[{"name":"column0","type":["StructType",[["k1",["DataType","Int32"]],["k2",["DataType","Int64"]]]]}]})___"; + + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Void) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + column.mutable_type()->set_void_type(google::protobuf::NullValue()); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + Y_UNUSED(cell); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + // Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":"Void"}],"columns":[{"name":"column0","type":["VoidType"]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Null) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + column.mutable_type()->set_null_type(google::protobuf::NullValue()); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + Y_UNUSED(cell); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + // Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":null}],"columns":[{"name":"column0","type":["NullType"]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(EmptyList) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + column.mutable_type()->set_empty_list_type(google::protobuf::NullValue()); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + Y_UNUSED(cell); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + // Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":[]}],"columns":[{"name":"column0","type":["EmptyListType"]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(EmptyDict) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + column.mutable_type()->set_empty_dict_type(google::protobuf::NullValue()); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + Y_UNUSED(cell); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + // Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":[]}],"columns":[{"name":"column0","type":["EmptyDictType"]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Tuple) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& tuple_type = *column.mutable_type()->mutable_tuple_type(); + tuple_type.add_elements()->set_type_id(Ydb::Type::INT32); + tuple_type.add_elements()->set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.add_items()->set_int32_value(31337); // k1 + cell.add_items()->set_int64_value(113370); // k2 + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + //Cerr << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":["31337","113370"]}],"columns":[{"name":"column0","type":["TupleType",[["DataType","Int32"],["DataType","Int64"]]]}]})___"; + + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Dict) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& dict_type = *column.mutable_type()->mutable_dict_type(); + dict_type.mutable_key()->set_type_id(Ydb::Type::INT32); + dict_type.mutable_payload()->set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + auto* pair = cell.add_pairs(); + pair->mutable_key()->set_int32_value(31337); + pair->mutable_payload()->set_int64_value(113370); + + pair = cell.add_pairs(); + pair->mutable_key()->set_int32_value(113370); + pair->mutable_payload()->set_int64_value(31337); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + + //Cerr << stream.Str() << Endl; TString expected1 = R"___({"data":[{"column0":[["31337","113370"],["113370","31337"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___"; TString expected2 = R"___({"data":[{"column0":[["113370","31337"],["31337","113370"]]}],"columns":[{"name":"column0","type":["DictType",["DataType","Int32"],["DataType","Int64"]]}]})___"; - + auto actual = stream.Str(); UNIT_ASSERT_C(actual == expected1 || actual == expected2, "expected either " << expected1 << " or " << expected2 << ", got " << actual); - } - - Y_UNIT_TEST(VariantTuple) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& variant_type = *column.mutable_type()->mutable_variant_type(); - auto& tuple_items = *variant_type.mutable_tuple_items(); - tuple_items.add_elements()->set_type_id(Ydb::Type::INT32); - tuple_items.add_elements()->set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.set_variant_index(0); - cell.set_int32_value(31337); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.set_variant_index(1); - cell.set_int64_value(131337); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - // Cerr << "Stream >> " << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":["0","31337"]},{"column0":["1","131337"]}],"columns":[{"name":"column0","type":["VariantType",["TupleType",[["DataType","Int32"],["DataType","Int64"]]]]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(VariantStruct) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& variant_type = *column.mutable_type()->mutable_variant_type(); - auto& struct_items = *variant_type.mutable_struct_items(); - - auto& m1 = *struct_items.add_members(); - auto& m2 = *struct_items.add_members(); - - m1.set_name("k1"); - m1.mutable_type()->set_type_id(Ydb::Type::INT32); - - m2.set_name("k2"); - m2.mutable_type()->set_type_id(Ydb::Type::INT64); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.set_variant_index(0); - cell.set_int32_value(31337); - } - { - auto& value = *rs.add_rows(); - auto& cell = *value.add_items(); - cell.set_variant_index(1); - cell.set_int64_value(131337); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - //Cerr << "Stream >> " << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":["0","31337"]},{"column0":["1","131337"]}],"columns":[{"name":"column0","type":["VariantType",["StructType",[["k1",["DataType","Int32"]],["k2",["DataType","Int64"]]]]]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - - Y_UNIT_TEST(Tagged) { - Ydb::ResultSet rs; - { - auto& column = *rs.add_columns(); - column.set_name("column0"); - auto& tagged_type = *column.mutable_type()->mutable_tagged_type(); - tagged_type.set_tag("tag"); - auto& type = *tagged_type.mutable_type(); - type.set_type_id(Ydb::Type::INT32); - } - { - auto& value = *rs.add_rows(); - value.add_items()->set_int32_value(31337); - } - - NJson::TJsonValue root; - FormatResultSet(root, rs); - - TStringStream stream; - NJson::WriteJson(&stream, &root); - // Cerr << "Stream >> " << stream.Str() << Endl; - TString expected = R"___({"data":[{"column0":"31337"}],"columns":[{"name":"column0","type":["TaggedType","tag",["DataType","Int32"]]}]})___"; - UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); - } - + } + + Y_UNIT_TEST(VariantTuple) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& variant_type = *column.mutable_type()->mutable_variant_type(); + auto& tuple_items = *variant_type.mutable_tuple_items(); + tuple_items.add_elements()->set_type_id(Ydb::Type::INT32); + tuple_items.add_elements()->set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.set_variant_index(0); + cell.set_int32_value(31337); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.set_variant_index(1); + cell.set_int64_value(131337); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + // Cerr << "Stream >> " << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":["0","31337"]},{"column0":["1","131337"]}],"columns":[{"name":"column0","type":["VariantType",["TupleType",[["DataType","Int32"],["DataType","Int64"]]]]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(VariantStruct) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& variant_type = *column.mutable_type()->mutable_variant_type(); + auto& struct_items = *variant_type.mutable_struct_items(); + + auto& m1 = *struct_items.add_members(); + auto& m2 = *struct_items.add_members(); + + m1.set_name("k1"); + m1.mutable_type()->set_type_id(Ydb::Type::INT32); + + m2.set_name("k2"); + m2.mutable_type()->set_type_id(Ydb::Type::INT64); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.set_variant_index(0); + cell.set_int32_value(31337); + } + { + auto& value = *rs.add_rows(); + auto& cell = *value.add_items(); + cell.set_variant_index(1); + cell.set_int64_value(131337); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + //Cerr << "Stream >> " << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":["0","31337"]},{"column0":["1","131337"]}],"columns":[{"name":"column0","type":["VariantType",["StructType",[["k1",["DataType","Int32"]],["k2",["DataType","Int64"]]]]]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + + Y_UNIT_TEST(Tagged) { + Ydb::ResultSet rs; + { + auto& column = *rs.add_columns(); + column.set_name("column0"); + auto& tagged_type = *column.mutable_type()->mutable_tagged_type(); + tagged_type.set_tag("tag"); + auto& type = *tagged_type.mutable_type(); + type.set_type_id(Ydb::Type::INT32); + } + { + auto& value = *rs.add_rows(); + value.add_items()->set_int32_value(31337); + } + + NJson::TJsonValue root; + FormatResultSet(root, rs); + + TStringStream stream; + NJson::WriteJson(&stream, &root); + // Cerr << "Stream >> " << stream.Str() << Endl; + TString expected = R"___({"data":[{"column0":"31337"}],"columns":[{"name":"column0","type":["TaggedType","tag",["DataType","Int32"]]}]})___"; + UNIT_ASSERT_VALUES_EQUAL(stream.Str(), expected); + } + Y_UNIT_TEST(FormatEmptySchema) { YandexQuery::Schema s; auto result = FormatSchema(s); @@ -414,4 +414,4 @@ Y_UNIT_TEST_SUITE(ResultFormatter) { auto result = FormatSchema(s); UNIT_ASSERT_VALUES_EQUAL(result, R"(["StructType";[["key";["DataType";"String"]];["value";["OptionalType";["DataType";"Uint64"]]]]])"); } -} +} diff --git a/ydb/core/yq/libs/shared_resources/db_pool.cpp b/ydb/core/yq/libs/shared_resources/db_pool.cpp index 570cee7f4c..372dfd72cd 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.cpp +++ b/ydb/core/yq/libs/shared_resources/db_pool.cpp @@ -1,68 +1,68 @@ -#include "db_pool.h" - +#include "db_pool.h" + #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/actor_bootstrapped.h> - + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/actor_bootstrapped.h> + #include <util/stream/file.h> #include <util/string/strip.h> #define LOG_E(stream) \ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, stream) - + namespace NYq { - -using namespace NActors; -using namespace NYql; - -class TDbPoolActor : public NActors::TActor<TDbPoolActor> { -public: + +using namespace NActors; +using namespace NYql; + +class TDbPoolActor : public NActors::TActor<TDbPoolActor> { +public: TDbPoolActor( const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters) - : TActor(&TThis::WorkingState) - , TableClient(tableClient) + : TActor(&TThis::WorkingState) + , TableClient(tableClient) , QueueSize(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("InFlight", NMonitoring::ExponentialHistogram(10, 2, 10))) , TotalInFlight(counters->GetSubgroup("subcomponent", "DbPool")->GetCounter("TotalInflight")) , RequestsTime(counters->GetSubgroup("subcomponent", "DbPool")->GetHistogram("RequestTimeMs", NMonitoring::ExponentialHistogram(6, 3, 100))) - {} - + {} + static constexpr char ActorName[] = "YQ_DB_POOL"; - - STRICT_STFUNC(WorkingState, - CFunc(NActors::TEvents::TEvPoison::EventType, Die) - HFunc(TEvents::TEvDbRequest, HandleRequest) - HFunc(TEvents::TEvDbResponse, HandleResponse) + + STRICT_STFUNC(WorkingState, + CFunc(NActors::TEvents::TEvPoison::EventType, Die) + HFunc(TEvents::TEvDbRequest, HandleRequest) + HFunc(TEvents::TEvDbResponse, HandleResponse) HFunc(TEvents::TEvDbFunctionRequest, HandleRequest) HFunc(TEvents::TEvDbFunctionResponse, HandleResponse) - ) - - void Die(const TActorContext& ctx) override { - NYql::TIssues issues; - issues.AddIssue("DB connection closed"); - auto cancelled = NYdb::TStatus(NYdb::EStatus::CANCELLED, std::move(issues)); - for (const auto& x : Requests) { + ) + + void Die(const TActorContext& ctx) override { + NYql::TIssues issues; + issues.AddIssue("DB connection closed"); + auto cancelled = NYdb::TStatus(NYdb::EStatus::CANCELLED, std::move(issues)); + for (const auto& x : Requests) { if (auto pRequest = std::get_if<TRequest>(&x)) { ctx.Send(pRequest->Sender, new TEvents::TEvDbResponse(cancelled, {})); } else if (auto pRequest = std::get_if<TFunctionRequest>(&x)) { ctx.Send(pRequest->Sender, new TEvents::TEvDbFunctionResponse(cancelled)); } - } - - State.reset(); - IActor::Die(ctx); - } - - void ProcessQueue(const TActorContext& ctx) { + } + + State.reset(); + IActor::Die(ctx); + } + + void ProcessQueue(const TActorContext& ctx) { QueueSize->Collect(Requests.size()); - if (Requests.empty() || RequestInProgress) { - return; - } + if (Requests.empty() || RequestInProgress) { + return; + } TotalInFlight->Inc(); - - RequestInProgress = true; + + RequestInProgress = true; RequestInProgressTimestamp = TInstant::Now(); const auto& requestVariant = Requests.front(); @@ -102,22 +102,22 @@ public: } }); } - } - - void HandleRequest(TEvents::TEvDbRequest::TPtr& ev, const TActorContext& ctx) { - auto request = ev->Get(); + } + + void HandleRequest(TEvents::TEvDbRequest::TPtr& ev, const TActorContext& ctx) { + auto request = ev->Get(); Requests.emplace_back(TRequest{ev->Sender, ev->Cookie, request->Sql, std::move(request->Params), request->Idempotent}); - ProcessQueue(ctx); - } - + ProcessQueue(ctx); + } + void PopFromQueueAndProcess(const TActorContext& ctx) { - RequestInProgress = false; + RequestInProgress = false; RequestsTime->Collect(TInstant::Now().MilliSeconds() - RequestInProgressTimestamp.MilliSeconds()); - Requests.pop_front(); + Requests.pop_front(); TotalInFlight->Dec(); - ProcessQueue(ctx); - } - + ProcessQueue(ctx); + } + void HandleResponse(TEvents::TEvDbResponse::TPtr& ev, const TActorContext& ctx) { const auto& request = Requests.front(); ctx.Send(ev->Forward(std::visit([](const auto& arg) { return arg.Sender; }, request))); @@ -136,24 +136,24 @@ public: PopFromQueueAndProcess(ctx); } -private: - struct TRequest { - TActorId Sender; - ui64 Cookie; - TString Sql; - NYdb::TParams Params; - bool Idempotent; - - TRequest() = default; - TRequest(const TActorId sender, ui64 cookie, const TString& sql, NYdb::TParams&& params, bool idempotent) - : Sender(sender) - , Cookie(cookie) - , Sql(sql) - , Params(std::move(params)) - , Idempotent(idempotent) - {} - }; - +private: + struct TRequest { + TActorId Sender; + ui64 Cookie; + TString Sql; + NYdb::TParams Params; + bool Idempotent; + + TRequest() = default; + TRequest(const TActorId sender, ui64 cookie, const TString& sql, NYdb::TParams&& params, bool idempotent) + : Sender(sender) + , Cookie(cookie) + , Sql(sql) + , Params(std::move(params)) + , Idempotent(idempotent) + {} + }; + struct TFunctionRequest { using TFunction = std::function<NYdb::TAsyncStatus(NYdb::NTable::TSession&)>; TActorId Sender; @@ -168,49 +168,49 @@ private: {} }; - NYdb::NTable::TTableClient TableClient; + NYdb::NTable::TTableClient TableClient; TDeque<std::variant<TRequest, TFunctionRequest>> Requests; - bool RequestInProgress = false; + bool RequestInProgress = false; TInstant RequestInProgressTimestamp = TInstant::Now(); - std::shared_ptr<int> State = std::make_shared<int>(); + std::shared_ptr<int> State = std::make_shared<int>(); const NMonitoring::THistogramPtr QueueSize; const NMonitoring::TDynamicCounters::TCounterPtr TotalInFlight; const NMonitoring::THistogramPtr RequestsTime; -}; - +}; + TDbPool::TDbPool( ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters) -{ - const auto& ctx = NActors::TActivationContext::AsActorContext(); - auto parentId = ctx.SelfID; - Actors.reserve(sessionsCount); - for (ui32 i = 0; i < sessionsCount; ++i) { - Actors.emplace_back( - NActors::TActivationContext::Register( +{ + const auto& ctx = NActors::TActivationContext::AsActorContext(); + auto parentId = ctx.SelfID; + Actors.reserve(sessionsCount); + for (ui32 i = 0; i < sessionsCount; ++i) { + Actors.emplace_back( + NActors::TActivationContext::Register( new TDbPoolActor(tableClient, counters), - parentId, NActors::TMailboxType::HTSwap, parentId.PoolID())); - } -} - -void TDbPool::Cleanup() { - auto parentId = NActors::TActivationContext::AsActorContext().SelfID; - for (const auto& actorId : Actors) { - NActors::TActivationContext::Send(new IEventHandle(actorId, parentId, new NActors::TEvents::TEvPoison())); - } -} - -TActorId TDbPool::GetNextActor() { - TGuard<TMutex> lock(Mutex); - Y_ENSURE(!Actors.empty()); - if (Index >= Actors.size()) { - Index = 0; - } - - return Actors[Index++]; -} - + parentId, NActors::TMailboxType::HTSwap, parentId.PoolID())); + } +} + +void TDbPool::Cleanup() { + auto parentId = NActors::TActivationContext::AsActorContext().SelfID; + for (const auto& actorId : Actors) { + NActors::TActivationContext::Send(new IEventHandle(actorId, parentId, new NActors::TEvents::TEvPoison())); + } +} + +TActorId TDbPool::GetNextActor() { + TGuard<TMutex> lock(Mutex); + Y_ENSURE(!Actors.empty()); + if (Index >= Actors.size()) { + Index = 0; + } + + return Actors[Index++]; +} + static void PrepareConfig(NYq::NConfig::TDbPoolConfig& config) { if (!config.GetStorage().GetToken() && config.GetStorage().GetOAuthFile()) { config.MutableStorage()->SetToken(StripString(TFileInput(config.GetStorage().GetOAuthFile()).ReadAll())); @@ -226,66 +226,66 @@ TDbPoolMap::TDbPoolMap( NYdb::TDriver driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) - : Config(config) - , Driver(driver) + : Config(config) + , Driver(driver) , CredentialsProviderFactory(credentialsProviderFactory) , Counters(counters) { PrepareConfig(Config); } - + TDbPoolHolder::TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, const NYdb::TDriver& driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters) - : Driver(driver) + : Driver(driver) , Pools(new TDbPoolMap(config, Driver, credentialsProviderFactory, counters)) -{ } - -TDbPoolHolder::~TDbPoolHolder() -{ - Driver.Stop(true); -} - +{ } + +TDbPoolHolder::~TDbPoolHolder() +{ + Driver.Stop(true); +} + void TDbPoolHolder::Reset(const NYq::NConfig::TDbPoolConfig& config) { - Pools->Reset(config); -} - -TDbPoolMap::TPtr TDbPoolHolder::Get() { - return Pools; -} - + Pools->Reset(config); +} + +TDbPoolMap::TPtr TDbPoolHolder::Get() { + return Pools; +} + void TDbPoolMap::Reset(const NYq::NConfig::TDbPoolConfig& config) { - TGuard<TMutex> lock(Mutex); - Config = config; + TGuard<TMutex> lock(Mutex); + Config = config; PrepareConfig(Config); - Pools.clear(); - TableClient = nullptr; -} - -TDbPool::TPtr TDbPoolHolder::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { - return Pools->GetOrCreate(dbPoolId, sessionsCount); -} - -TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { - TGuard<TMutex> lock(Mutex); - auto it = Pools.find(dbPoolId); - if (it != Pools.end()) { - return it->second; - } - + Pools.clear(); + TableClient = nullptr; +} + +TDbPool::TPtr TDbPoolHolder::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { + return Pools->GetOrCreate(dbPoolId, sessionsCount); +} + +TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { + TGuard<TMutex> lock(Mutex); + auto it = Pools.find(dbPoolId); + if (it != Pools.end()) { + return it->second; + } + if (!Config.GetStorage().GetEndpoint()) { - return nullptr; - } - - if (!TableClient) { - auto clientSettings = NYdb::NTable::TClientSettings() - .UseQueryCache(false) + return nullptr; + } + + if (!TableClient) { + auto clientSettings = NYdb::NTable::TClientSettings() + .UseQueryCache(false) .SessionPoolSettings(NYdb::NTable::TSessionPoolSettings().MaxActiveSessions(1 + Config.GetMaxSessionCount())) .Database(Config.GetStorage().GetDatabase()) .DiscoveryEndpoint(Config.GetStorage().GetEndpoint()) - .DiscoveryMode(NYdb::EDiscoveryMode::Async); + .DiscoveryMode(NYdb::EDiscoveryMode::Async); NKikimr::TYdbCredentialsSettings credSettings; credSettings.UseLocalMetadata = Config.GetStorage().GetUseLocalMetadataService(); credSettings.OAuthToken = Config.GetStorage().GetToken(); @@ -293,17 +293,17 @@ TDbPool::TPtr TDbPoolMap::GetOrCreate(EDbPoolId dbPoolId, ui32 sessionsCount) { clientSettings.CredentialsProviderFactory(CredentialsProviderFactory(credSettings)); clientSettings.EnableSsl(Config.GetStorage().GetUseSsl()); - - TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings); - } - + + TableClient = MakeHolder<NYdb::NTable::TTableClient>(Driver, clientSettings); + } + TDbPool::TPtr dbPool = new TDbPool(sessionsCount, *TableClient, Counters); - Pools.emplace(dbPoolId, dbPool); - return dbPool; -} - -NYdb::TDriver& TDbPoolHolder::GetDriver() { - return Driver; -} - + Pools.emplace(dbPoolId, dbPool); + return dbPool; +} + +NYdb::TDriver& TDbPoolHolder::GetDriver() { + return Driver; +} + } /* namespace NYq */ diff --git a/ydb/core/yq/libs/shared_resources/db_pool.h b/ydb/core/yq/libs/shared_resources/db_pool.h index 878236f917..68d9adc2fc 100644 --- a/ydb/core/yq/libs/shared_resources/db_pool.h +++ b/ydb/core/yq/libs/shared_resources/db_pool.h @@ -1,85 +1,85 @@ -#pragma once +#pragma once #include <ydb/core/yq/libs/config/protos/yq_config.pb.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/core/yq/libs/events/events.h> - + #include <ydb/library/security/ydb_credentials_provider_factory.h> -#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/actor.h> #include <library/cpp/monlib/dynamic_counters/counters.h> - + #include <util/system/mutex.h> namespace NYq { - -class TDbPool: public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TDbPool>; - - void Cleanup(); - - NActors::TActorId GetNextActor(); - -private: - friend class TDbPoolMap; - + +class TDbPool: public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TDbPool>; + + void Cleanup(); + + NActors::TActorId GetNextActor(); + +private: + friend class TDbPoolMap; + TDbPool(ui32 sessionsCount, const NYdb::NTable::TTableClient& tableClient, const NMonitoring::TDynamicCounterPtr& counters); - - TMutex Mutex; - TVector<NActors::TActorId> Actors; - ui32 Index = 0; + + TMutex Mutex; + TVector<NActors::TActorId> Actors; + ui32 Index = 0; const NMonitoring::TDynamicCounterPtr Counters; -}; - -enum class EDbPoolId { - MAIN = 0, - REFRESH = 1 -}; - -class TDbPoolMap: public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TDbPoolMap>; - - TDbPool::TPtr GetOrCreate(EDbPoolId poolId, ui32 sessionsCount); - -private: - friend class TDbPoolHolder; - +}; + +enum class EDbPoolId { + MAIN = 0, + REFRESH = 1 +}; + +class TDbPoolMap: public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TDbPoolMap>; + + TDbPool::TPtr GetOrCreate(EDbPoolId poolId, ui32 sessionsCount); + +private: + friend class TDbPoolHolder; + TDbPoolMap(const NYq::NConfig::TDbPoolConfig& config, NYdb::TDriver driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); void Reset(const NYq::NConfig::TDbPoolConfig& config); - TMutex Mutex; + TMutex Mutex; NYq::NConfig::TDbPoolConfig Config; - NYdb::TDriver Driver; - THashMap<EDbPoolId, TDbPool::TPtr> Pools; - THolder<NYdb::NTable::TTableClient> TableClient; + NYdb::TDriver Driver; + THashMap<EDbPoolId, TDbPool::TPtr> Pools; + THolder<NYdb::NTable::TTableClient> TableClient; NKikimr::TYdbCredentialsProviderFactory CredentialsProviderFactory; const NMonitoring::TDynamicCounterPtr Counters; -}; - -class TDbPoolHolder: public TThrRefBase { -public: - using TPtr = TIntrusivePtr<TDbPoolHolder>; +}; + +class TDbPoolHolder: public TThrRefBase { +public: + using TPtr = TIntrusivePtr<TDbPoolHolder>; TDbPoolHolder( const NYq::NConfig::TDbPoolConfig& config, const NYdb::TDriver& driver, const NKikimr::TYdbCredentialsProviderFactory& credentialsProviderFactory, const NMonitoring::TDynamicCounterPtr& counters); - ~TDbPoolHolder(); - + ~TDbPoolHolder(); + void Reset(const NYq::NConfig::TDbPoolConfig& config); - TDbPool::TPtr GetOrCreate(EDbPoolId poolId, ui32 sessionsCount); - NYdb::TDriver& GetDriver(); - TDbPoolMap::TPtr Get(); - -public: - NYdb::TDriver Driver; - TDbPoolMap::TPtr Pools; -}; - + TDbPool::TPtr GetOrCreate(EDbPoolId poolId, ui32 sessionsCount); + NYdb::TDriver& GetDriver(); + TDbPoolMap::TPtr Get(); + +public: + NYdb::TDriver Driver; + TDbPoolMap::TPtr Pools; +}; + } /* NYq */ diff --git a/ydb/core/yq/libs/tasks_packer/tasks_packer.cpp b/ydb/core/yq/libs/tasks_packer/tasks_packer.cpp index 2b485063c9..f7bbd04c06 100644 --- a/ydb/core/yq/libs/tasks_packer/tasks_packer.cpp +++ b/ydb/core/yq/libs/tasks_packer/tasks_packer.cpp @@ -1,54 +1,54 @@ -#include "tasks_packer.h" - -#include <ydb/library/yql/utils/yql_panic.h> - -namespace NYq { - -namespace NTasksPacker { - -void Pack(TVector<NYql::NDqProto::TDqTask>& tasks, THashMap<i64, TString>& stagePrograms) { - for (auto& task : tasks) { - auto stageId = task.GetStageId(); - auto it = stagePrograms.find(stageId); - if (it == stagePrograms.end()) { - stagePrograms[stageId] = std::move(*task.MutableProgram()->MutableRaw()); - task.MutableProgram()->MutableRaw()->clear(); - } - } -} - -void UnPack(TVector<NYql::NDqProto::TDqTask>& tasks, const THashMap<i64, TString>& stagePrograms) { - for (auto& task : tasks) { - auto stageId = task.GetStageId(); - auto it = stagePrograms.find(stageId); - YQL_ENSURE(it != stagePrograms.end()); - *task.MutableProgram()->MutableRaw() = it->second; - } -} - -void UnPack( - google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& dst, - const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& src, - const google::protobuf::Map<i64, TString>& stagePrograms) -{ - if (stagePrograms.empty()) { - dst = src; - return; - } - TVector<NYql::NDqProto::TDqTask> tasks; - tasks.reserve(src.size()); - for (const auto& srcTask : src) { - auto stageId = srcTask.GetStageId(); - auto it = stagePrograms.find(stageId); - YQL_ENSURE(it != stagePrograms.end()); - - NYql::NDqProto::TDqTask task = srcTask; - *task.MutableProgram()->MutableRaw() = it->second; - tasks.emplace_back(task); - } - dst.Assign(tasks.begin(), tasks.end()); -} - -} // namespace NTasksPacker - -} // namespace NYq +#include "tasks_packer.h" + +#include <ydb/library/yql/utils/yql_panic.h> + +namespace NYq { + +namespace NTasksPacker { + +void Pack(TVector<NYql::NDqProto::TDqTask>& tasks, THashMap<i64, TString>& stagePrograms) { + for (auto& task : tasks) { + auto stageId = task.GetStageId(); + auto it = stagePrograms.find(stageId); + if (it == stagePrograms.end()) { + stagePrograms[stageId] = std::move(*task.MutableProgram()->MutableRaw()); + task.MutableProgram()->MutableRaw()->clear(); + } + } +} + +void UnPack(TVector<NYql::NDqProto::TDqTask>& tasks, const THashMap<i64, TString>& stagePrograms) { + for (auto& task : tasks) { + auto stageId = task.GetStageId(); + auto it = stagePrograms.find(stageId); + YQL_ENSURE(it != stagePrograms.end()); + *task.MutableProgram()->MutableRaw() = it->second; + } +} + +void UnPack( + google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& dst, + const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& src, + const google::protobuf::Map<i64, TString>& stagePrograms) +{ + if (stagePrograms.empty()) { + dst = src; + return; + } + TVector<NYql::NDqProto::TDqTask> tasks; + tasks.reserve(src.size()); + for (const auto& srcTask : src) { + auto stageId = srcTask.GetStageId(); + auto it = stagePrograms.find(stageId); + YQL_ENSURE(it != stagePrograms.end()); + + NYql::NDqProto::TDqTask task = srcTask; + *task.MutableProgram()->MutableRaw() = it->second; + tasks.emplace_back(task); + } + dst.Assign(tasks.begin(), tasks.end()); +} + +} // namespace NTasksPacker + +} // namespace NYq diff --git a/ydb/core/yq/libs/tasks_packer/tasks_packer.h b/ydb/core/yq/libs/tasks_packer/tasks_packer.h index f603dc1823..2fb8dfdcec 100644 --- a/ydb/core/yq/libs/tasks_packer/tasks_packer.h +++ b/ydb/core/yq/libs/tasks_packer/tasks_packer.h @@ -1,21 +1,21 @@ -#pragma once - -#include <ydb/library/yql/dq/proto/dq_tasks.pb.h> - -#include <util/generic/vector.h> -#include <util/generic/hash.h> - -namespace NYq { - -namespace NTasksPacker { - -void Pack(TVector<NYql::NDqProto::TDqTask>& tasks, THashMap<i64, TString>& stagePrograms); -void UnPack(TVector<NYql::NDqProto::TDqTask>& tasks, const THashMap<i64, TString>& stagePrograms); -void UnPack( - google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& dst, - const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& src, - const google::protobuf::Map<i64, TString>& stagePrograms); - -} // namespace NTasksPacker - -} // namespace NYq +#pragma once + +#include <ydb/library/yql/dq/proto/dq_tasks.pb.h> + +#include <util/generic/vector.h> +#include <util/generic/hash.h> + +namespace NYq { + +namespace NTasksPacker { + +void Pack(TVector<NYql::NDqProto::TDqTask>& tasks, THashMap<i64, TString>& stagePrograms); +void UnPack(TVector<NYql::NDqProto::TDqTask>& tasks, const THashMap<i64, TString>& stagePrograms); +void UnPack( + google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& dst, + const google::protobuf::RepeatedPtrField<NYql::NDqProto::TDqTask>& src, + const google::protobuf::Map<i64, TString>& stagePrograms); + +} // namespace NTasksPacker + +} // namespace NYq diff --git a/ydb/core/yq/libs/tasks_packer/ya.make b/ydb/core/yq/libs/tasks_packer/ya.make index 65e04275b1..4272569871 100644 --- a/ydb/core/yq/libs/tasks_packer/ya.make +++ b/ydb/core/yq/libs/tasks_packer/ya.make @@ -1,16 +1,16 @@ -OWNER(g:yq) - -LIBRARY() - -SRCS( - tasks_packer.cpp -) - -PEERDIR( - ydb/library/yql/dq/proto - ydb/library/yql/utils -) - -YQL_LAST_ABI_VERSION() - -END() +OWNER(g:yq) + +LIBRARY() + +SRCS( + tasks_packer.cpp +) + +PEERDIR( + ydb/library/yql/dq/proto + ydb/library/yql/utils +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/yq/libs/ya.make b/ydb/core/yq/libs/ya.make index 60392384e0..ccc88a55b6 100644 --- a/ydb/core/yq/libs/ya.make +++ b/ydb/core/yq/libs/ya.make @@ -18,14 +18,14 @@ RECURSE( hmac init logs - mock + mock pretty_printers private_client read_rule result_formatter shared_resources signer - tasks_packer + tasks_packer test_connection ydb ) diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index e0188c5b9c..53d7722dee 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -222,18 +222,18 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { res.set_void_type(::google::protobuf::NULL_VALUE); break; - case TType::EKind::Null: - res.set_null_type(::google::protobuf::NULL_VALUE); - break; - - case TType::EKind::EmptyList: - res.set_empty_list_type(::google::protobuf::NULL_VALUE); - break; - - case TType::EKind::EmptyDict: - res.set_empty_dict_type(::google::protobuf::NULL_VALUE); - break; - + case TType::EKind::Null: + res.set_null_type(::google::protobuf::NULL_VALUE); + break; + + case TType::EKind::EmptyList: + res.set_empty_list_type(::google::protobuf::NULL_VALUE); + break; + + case TType::EKind::EmptyDict: + res.set_empty_dict_type(::google::protobuf::NULL_VALUE); + break; + case TType::EKind::Data: { auto dataType = static_cast<TDataType*>(type); auto schemeType = dataType->GetSchemeType(); @@ -317,15 +317,15 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res) { break; } - case TType::EKind::Tagged: { - auto taggedType = static_cast<TTaggedType*>(type); - TType* innerType = taggedType->GetBaseType(); - auto& resType = *res.mutable_tagged_type(); - resType.set_tag(TString(taggedType->GetTag())); - ExportTypeToProtoImpl(innerType, *resType.mutable_type()); - break; - } - + case TType::EKind::Tagged: { + auto taggedType = static_cast<TTaggedType*>(type); + TType* innerType = taggedType->GetBaseType(); + auto& resType = *res.mutable_tagged_type(); + resType.set_tag(TString(taggedType->GetTag())); + ExportTypeToProtoImpl(innerType, *resType.mutable_type()); + break; + } + default: MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); } @@ -615,12 +615,12 @@ void ExportValueToProtoImpl(TType* type, const NUdf::TUnboxedValuePod& value, Yd break; } - case TType::EKind::Tagged: { - auto taggedType = static_cast<TTaggedType*>(type); - ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); - break; - } - + case TType::EKind::Tagged: { + auto taggedType = static_cast<TTaggedType*>(type); + ExportValueToProtoImpl(taggedType->GetBaseType(), value, res); + break; + } + default: { MKQL_ENSURE(false, TStringBuilder() << "Unknown kind: " << type->GetKindAsStr()); } diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp index 7999166c6b..b74abc1c34 100644 --- a/ydb/library/yql/core/facade/yql_facade.cpp +++ b/ydb/library/yql/core/facade/yql_facade.cpp @@ -665,7 +665,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsync( ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e)); return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - return AsyncTransformWithFallback(false); + return AsyncTransformWithFallback(false); }); } @@ -734,7 +734,7 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig( ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e)); return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - return AsyncTransformWithFallback(false); + return AsyncTransformWithFallback(false); }); } @@ -805,8 +805,8 @@ TProgram::TFutureStatus TProgram::RunAsync( return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - SavedExprRoot_ = ExprRoot_; - + SavedExprRoot_ = ExprRoot_; + return openSession.Apply([this](const TFuture<void>& f) { YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { @@ -816,7 +816,7 @@ TProgram::TFutureStatus TProgram::RunAsync( ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e)); return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - return AsyncTransformWithFallback(false); + return AsyncTransformWithFallback(false); }); } @@ -877,8 +877,8 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig( return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - SavedExprRoot_ = ExprRoot_; - + SavedExprRoot_ = ExprRoot_; + return openSession.Apply([this](const TFuture<void>& f) { YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); try { @@ -888,115 +888,115 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig( ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e)); return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error); } - return AsyncTransformWithFallback(false); + return AsyncTransformWithFallback(false); }); } -TFuture<IGraphTransformer::TStatus> TProgram::AsyncTransformWithFallback(bool applyAsyncChanges) -{ - return AsyncTransform(*Transformer_, ExprRoot_, *ExprCtx_, applyAsyncChanges).Apply([this](const TFuture<IGraphTransformer::TStatus>& res) { - auto status = res.GetValueSync(); - if (status == IGraphTransformer::TStatus::Error - && !TypeCtx_->ForceDq - && SavedExprRoot_ - && TypeCtx_->DqCaptured - && TypeCtx_->DqFallbackPolicy != "never") - { - ExprRoot_ = SavedExprRoot_; - SavedExprRoot_ = nullptr; - - auto issues = ExprCtx_->IssueManager.GetIssues(); - bool hasDqGatewayError = false; - bool hasDqGatewayFallbackError = false; - - auto checkIssue = [&](const TIssue& issue) { - if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_ERROR) { - YQL_LOG(DEBUG) << "Gateway Error " << issue; - hasDqGatewayError = true; - } else if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR) { - YQL_LOG(DEBUG) << "Gateway Fallback Error " << issue; - hasDqGatewayError = true; - hasDqGatewayFallbackError = true; - } - }; - - std::function<void(const TIssuePtr& issue)> recursiveCheck = [&](const TIssuePtr& issue) { - checkIssue(*issue); - for (const auto& subissue : issue->GetSubIssues()) { - recursiveCheck(subissue); - } - }; - - std::function<void(const TIssuePtr& issue)> toWarning = [&](const TIssuePtr& issue) { - if (issue->Severity == TSeverityIds::S_ERROR - || issue->Severity == TSeverityIds::S_FATAL - || issue->Severity == TSeverityIds::S_WARNING) - { - issue->Severity = TSeverityIds::S_INFO; - } - for (const auto& subissue : issue->GetSubIssues()) { - toWarning(subissue); - } - }; - - for (const auto& issue : issues) { - checkIssue(issue); - // check subissues - for (const auto& subissue : issue.GetSubIssues()) { - recursiveCheck(subissue); - } - } - - ExprCtx_->IssueManager.Reset(); - - if (hasDqGatewayError && !hasDqGatewayFallbackError && TypeCtx_->DqFallbackPolicy.find("always") == TString::npos) { - // unrecoverable error - ExprCtx_->IssueManager.AddIssues(issues); - return res; - } - - YQL_LOG(DEBUG) << "Fallback, Issues: " << issues.ToString(); - - ExprCtx_->Reset(); - Transformer_->Rewind(); - - for (auto sink : TypeCtx_->DataSinks) { - sink->Reset(); - } - for (auto source : TypeCtx_->DataSources) { - source->Reset(); - } +TFuture<IGraphTransformer::TStatus> TProgram::AsyncTransformWithFallback(bool applyAsyncChanges) +{ + return AsyncTransform(*Transformer_, ExprRoot_, *ExprCtx_, applyAsyncChanges).Apply([this](const TFuture<IGraphTransformer::TStatus>& res) { + auto status = res.GetValueSync(); + if (status == IGraphTransformer::TStatus::Error + && !TypeCtx_->ForceDq + && SavedExprRoot_ + && TypeCtx_->DqCaptured + && TypeCtx_->DqFallbackPolicy != "never") + { + ExprRoot_ = SavedExprRoot_; + SavedExprRoot_ = nullptr; + + auto issues = ExprCtx_->IssueManager.GetIssues(); + bool hasDqGatewayError = false; + bool hasDqGatewayFallbackError = false; + + auto checkIssue = [&](const TIssue& issue) { + if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_ERROR) { + YQL_LOG(DEBUG) << "Gateway Error " << issue; + hasDqGatewayError = true; + } else if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR) { + YQL_LOG(DEBUG) << "Gateway Fallback Error " << issue; + hasDqGatewayError = true; + hasDqGatewayFallbackError = true; + } + }; + + std::function<void(const TIssuePtr& issue)> recursiveCheck = [&](const TIssuePtr& issue) { + checkIssue(*issue); + for (const auto& subissue : issue->GetSubIssues()) { + recursiveCheck(subissue); + } + }; + + std::function<void(const TIssuePtr& issue)> toWarning = [&](const TIssuePtr& issue) { + if (issue->Severity == TSeverityIds::S_ERROR + || issue->Severity == TSeverityIds::S_FATAL + || issue->Severity == TSeverityIds::S_WARNING) + { + issue->Severity = TSeverityIds::S_INFO; + } + for (const auto& subissue : issue->GetSubIssues()) { + toWarning(subissue); + } + }; + + for (const auto& issue : issues) { + checkIssue(issue); + // check subissues + for (const auto& subissue : issue.GetSubIssues()) { + recursiveCheck(subissue); + } + } + + ExprCtx_->IssueManager.Reset(); + + if (hasDqGatewayError && !hasDqGatewayFallbackError && TypeCtx_->DqFallbackPolicy.find("always") == TString::npos) { + // unrecoverable error + ExprCtx_->IssueManager.AddIssues(issues); + return res; + } + + YQL_LOG(DEBUG) << "Fallback, Issues: " << issues.ToString(); + + ExprCtx_->Reset(); + Transformer_->Rewind(); + + for (auto sink : TypeCtx_->DataSinks) { + sink->Reset(); + } + for (auto source : TypeCtx_->DataSources) { + source->Reset(); + } CleanupLastSession(); - - if (hasDqGatewayError) { - TIssue warning("DQ cannot execute the query"); - warning.Severity = TSeverityIds::S_INFO; - - for (auto& issue : issues) { - TIssuePtr newIssue = new TIssue(issue); - if (newIssue->Severity == TSeverityIds::S_ERROR - || issue.Severity == TSeverityIds::S_FATAL - || issue.Severity == TSeverityIds::S_WARNING) - { - newIssue->Severity = TSeverityIds::S_INFO; - } - for (auto& subissue : newIssue->GetSubIssues()) { - toWarning(subissue); - } - warning.AddSubIssue(newIssue); - } - - ExprCtx_->IssueManager.AddIssues({warning}); - } - - // don't execute recapture again - ExprCtx_->Step.Done(TExprStep::Recapture); - return AsyncTransformWithFallback(false); - } - return res; - }); -} - + + if (hasDqGatewayError) { + TIssue warning("DQ cannot execute the query"); + warning.Severity = TSeverityIds::S_INFO; + + for (auto& issue : issues) { + TIssuePtr newIssue = new TIssue(issue); + if (newIssue->Severity == TSeverityIds::S_ERROR + || issue.Severity == TSeverityIds::S_FATAL + || issue.Severity == TSeverityIds::S_WARNING) + { + newIssue->Severity = TSeverityIds::S_INFO; + } + for (auto& subissue : newIssue->GetSubIssues()) { + toWarning(subissue); + } + warning.AddSubIssue(newIssue); + } + + ExprCtx_->IssueManager.AddIssues({warning}); + } + + // don't execute recapture again + ExprCtx_->Step.Done(TExprStep::Recapture); + return AsyncTransformWithFallback(false); + } + return res; + }); +} + TMaybe<TString> TProgram::GetQueryAst() { if (ExternalQueryAst_) { return ExternalQueryAst_; @@ -1191,7 +1191,7 @@ TMaybe<TString> TProgram::GetDiscoveredData() { TProgram::TFutureStatus TProgram::ContinueAsync() { YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); - return AsyncTransformWithFallback(true); + return AsyncTransformWithFallback(true); } void TProgram::Abort() @@ -1202,7 +1202,7 @@ void TProgram::Abort() void TProgram::CleanupLastSession() { YQL_LOG_CTX_ROOT_SCOPE(GetSessionId()); - TString sessionId = GetSessionId(); + TString sessionId = GetSessionId(); if (sessionId.empty()) { return; } @@ -1320,8 +1320,8 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us if (providerNames.contains(DqProviderName)) { resultProviderDataSources.push_back(TString(DqProviderName)); - } - + } + if (!resultProviderDataSources.empty()) { auto resultFormat = ResultFormat_; diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h index 731ec3e29a..1f8f47681d 100644 --- a/ydb/library/yql/core/facade/yql_facade.h +++ b/ydb/library/yql/core/facade/yql_facade.h @@ -332,8 +332,8 @@ private: TString GetSessionId() const; TString TakeSessionId(); - NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransformWithFallback(bool applyAsyncChanges); - + NThreading::TFuture<IGraphTransformer::TStatus> AsyncTransformWithFallback(bool applyAsyncChanges); + private: const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_; const TIntrusivePtr<IRandomProvider> RandomProvider_; @@ -360,7 +360,7 @@ private: TAutoPtr<TExprContext> ExprCtx_; const IModuleResolver::TPtr Modules_; TExprNode::TPtr ExprRoot_; - TExprNode::TPtr SavedExprRoot_; + TExprNode::TPtr SavedExprRoot_; mutable TAdaptiveLock SessionIdLock_; TString SessionId_; TTypeAnnotationContextPtr TypeCtx_; diff --git a/ydb/library/yql/core/file_storage/file_storage.cpp b/ydb/library/yql/core/file_storage/file_storage.cpp index 96deba72d2..631adbc688 100644 --- a/ydb/library/yql/core/file_storage/file_storage.cpp +++ b/ydb/library/yql/core/file_storage/file_storage.cpp @@ -87,46 +87,46 @@ public: } TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) override { - YQL_LOG(INFO) << "PutFileStripped to cache: " << file; - if (originalMd5.empty()) { - YQL_LOG(WARN) << "Empty md5 for: " << file; - } - const auto md5 = originalMd5.empty() ? MD5::File(file) : originalMd5; - const auto strippedMetaFile = md5 + ".stripped_meta"; - auto lock = MultiResourceLock.Acquire(strippedMetaFile); - - TUrlMeta strippedMeta; - strippedMeta.TryReadFrom(GetRoot() / strippedMetaFile); - if (strippedMeta.ContentFile) { - if (auto result = Storage.HardlinkFromStorage(strippedMeta.ContentFile, strippedMeta.Md5, "")) { - return result; - } - } - - strippedMeta = TUrlMeta(); - const TString storageFileName = md5 + ".file.stripped"; - TFileLinkPtr result = Storage.Put(storageFileName, "", "", [&file](const TFsPath& dstPath) { + YQL_LOG(INFO) << "PutFileStripped to cache: " << file; + if (originalMd5.empty()) { + YQL_LOG(WARN) << "Empty md5 for: " << file; + } + const auto md5 = originalMd5.empty() ? MD5::File(file) : originalMd5; + const auto strippedMetaFile = md5 + ".stripped_meta"; + auto lock = MultiResourceLock.Acquire(strippedMetaFile); + + TUrlMeta strippedMeta; + strippedMeta.TryReadFrom(GetRoot() / strippedMetaFile); + if (strippedMeta.ContentFile) { + if (auto result = Storage.HardlinkFromStorage(strippedMeta.ContentFile, strippedMeta.Md5, "")) { + return result; + } + } + + strippedMeta = TUrlMeta(); + const TString storageFileName = md5 + ".file.stripped"; + TFileLinkPtr result = Storage.Put(storageFileName, "", "", [&file](const TFsPath& dstPath) { ui64 size; TString md5; TShellCommand cmd("strip", {file, "-o", dstPath.GetPath()}); cmd.Run().Wait(); if (*cmd.GetExitCode() != 0) { ythrow yexception() << cmd.GetError(); - } + } md5 = MD5::File(dstPath.GetPath()); size = TFile(dstPath.GetPath(), OpenExisting | RdOnly).GetLength(); YQL_LOG(DEBUG) << "Strip " << file << " to " << dstPath.GetPath(); return std::make_pair(size, md5); - }); - - strippedMeta.ContentFile = result->GetStorageFileName(); - strippedMeta.Md5 = result->GetMd5(); - auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); - strippedMeta.SaveTo(metaTmpFile); - Storage.MoveToStorage(metaTmpFile, strippedMetaFile); - return result; - } - + }); + + strippedMeta.ContentFile = result->GetStorageFileName(); + strippedMeta.Md5 = result->GetMd5(); + auto metaTmpFile = Storage.GetTemp() / Storage.GetTempName(); + strippedMeta.SaveTo(metaTmpFile); + Storage.MoveToStorage(metaTmpFile, strippedMetaFile); + return result; + } + TFileLinkPtr PutInline(const TString& data) override { const auto md5 = MD5::Calc(data); const TString storageFileName = md5 + ".file"; diff --git a/ydb/library/yql/core/file_storage/file_storage.h b/ydb/library/yql/core/file_storage/file_storage.h index edc73e3ae1..9133d57b1b 100644 --- a/ydb/library/yql/core/file_storage/file_storage.h +++ b/ydb/library/yql/core/file_storage/file_storage.h @@ -27,7 +27,7 @@ struct IFileStorage: public TThrRefBase { virtual ~IFileStorage() = default; virtual void AddDownloader(IDownloaderPtr downloader) = 0; virtual TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) = 0; - virtual TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) = 0; + virtual TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) = 0; virtual TFileLinkPtr PutInline(const TString& data) = 0; virtual TFileLinkPtr PutUrl(const TString& url, const TString& oauthToken) = 0; // async versions diff --git a/ydb/library/yql/core/file_storage/storage.cpp b/ydb/library/yql/core/file_storage/storage.cpp index d46b5b7365..247bff5242 100644 --- a/ydb/library/yql/core/file_storage/storage.cpp +++ b/ydb/library/yql/core/file_storage/storage.cpp @@ -253,10 +253,10 @@ public: TouchFile(storageFile.c_str()); SetCacheFilePermissionsNoThrow(hardlinkFile); - const i64 fileSize = GetFileLength(hardlinkFile); - if (fileSize < 0) { - ythrow yexception() << "Unable to get size for file " << hardlinkFile.GetPath().Quote(); - } + const i64 fileSize = GetFileLength(hardlinkFile); + if (fileSize < 0) { + ythrow yexception() << "Unable to get size for file " << hardlinkFile.GetPath().Quote(); + } TString md5 = storageFileMd5; if (!md5) { // could happen rarely diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index b050ad1a9d..a2ed91d640 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -167,10 +167,10 @@ message TIssuesIds { STAT_DEPRECATED_STRING_TREE = 5000; STAT_ACCESS_DENIED = 5001; -// dq - DQ_GATEWAY_ERROR = 6000; - DQ_GATEWAY_NEED_FALLBACK_ERROR = 6001; - +// dq + DQ_GATEWAY_ERROR = 6000; + DQ_GATEWAY_NEED_FALLBACK_ERROR = 6001; + // range [200000, 399999) reserved for KiKiMR issue codes, do not use! } diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index 8f5779ec44..897d5d6f9a 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -584,7 +584,7 @@ ids { code: YQL_JSON_QUERY_RETURNING_JSON_IS_DEPRECATED severity: S_WARNING } -ids { +ids { code: YQL_DEPRECATED_LIST_FLATMAP_OPTIONAL severity: S_WARNING } @@ -597,10 +597,10 @@ ids { severity: S_WARNING } ids { - code: DQ_GATEWAY_ERROR - severity: S_ERROR -} -ids { - code: DQ_GATEWAY_NEED_FALLBACK_ERROR - severity: S_ERROR -} + code: DQ_GATEWAY_ERROR + severity: S_ERROR +} +ids { + code: DQ_GATEWAY_NEED_FALLBACK_ERROR + severity: S_ERROR +} diff --git a/ydb/library/yql/core/services/yql_eval_expr.cpp b/ydb/library/yql/core/services/yql_eval_expr.cpp index 7f1f9e9762..811f22a605 100644 --- a/ydb/library/yql/core/services/yql_eval_expr.cpp +++ b/ydb/library/yql/core/services/yql_eval_expr.cpp @@ -359,7 +359,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - start"; bool pure = false; - TString nextProvider; + TString nextProvider; TMaybe<IDataProvider*> calcProvider; TExprNode::TPtr calcWorldRoot; bool isAtomPipeline = false; @@ -420,13 +420,13 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp if (calcTransfomer) { calcProvider.ConstructInPlace(); } else { - if (nextProvider.empty()) { - nextProvider = types.GetDefaultDataSource(); + if (nextProvider.empty()) { + nextProvider = types.GetDefaultDataSource(); + } + if (!nextProvider.empty() && + types.DataSourceMap.contains(nextProvider)) { + calcProvider = types.DataSourceMap[nextProvider].Get(); } - if (!nextProvider.empty() && - types.DataSourceMap.contains(nextProvider)) { - calcProvider = types.DataSourceMap[nextProvider].Get(); - } } } else if (!calcTransfomer) { for (auto& p : types.DataSources) { @@ -928,79 +928,79 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp clonedArg = ctx.NewCallable(clonedArg->Pos(), "SerializeCode", { clonedArg }); } - NYT::TNode ysonNode; - do { - calcProvider.Clear(); - calcWorldRoot.Drop(); - fullTransformer->Rewind(); - auto prevSteps = ctx.Step; - types.EvaluationInProgress = true; - status = SyncTransform(*fullTransformer, clonedArg, ctx); - ctx.Step = prevSteps; - types.EvaluationInProgress = false; - if (status.Level == IGraphTransformer::TStatus::Error) { - return nullptr; - } - - // execute calcWorldRoot - auto execTransformer = CreateExecutionTransformer(types, [](const TOperationProgress&){}, false); - status = SyncTransform(*execTransformer, calcWorldRoot, ctx); - if (status.Level == IGraphTransformer::TStatus::Error) { - return nullptr; - } - - IDataProvider::TFillSettings fillSettings; - auto delegatedNode = Build<TResult>(ctx, node->Pos()) - .Input(clonedArg) - .BytesLimit() - .Value(TString()) - .Build() - .RowsLimit() - .Value(TString()) - .Build() - .FormatDetails() + NYT::TNode ysonNode; + do { + calcProvider.Clear(); + calcWorldRoot.Drop(); + fullTransformer->Rewind(); + auto prevSteps = ctx.Step; + types.EvaluationInProgress = true; + status = SyncTransform(*fullTransformer, clonedArg, ctx); + ctx.Step = prevSteps; + types.EvaluationInProgress = false; + if (status.Level == IGraphTransformer::TStatus::Error) { + return nullptr; + } + + // execute calcWorldRoot + auto execTransformer = CreateExecutionTransformer(types, [](const TOperationProgress&){}, false); + status = SyncTransform(*execTransformer, calcWorldRoot, ctx); + if (status.Level == IGraphTransformer::TStatus::Error) { + return nullptr; + } + + IDataProvider::TFillSettings fillSettings; + auto delegatedNode = Build<TResult>(ctx, node->Pos()) + .Input(clonedArg) + .BytesLimit() + .Value(TString()) + .Build() + .RowsLimit() + .Value(TString()) + .Build() + .FormatDetails() .Value(ToString((ui32)NYson::EYsonFormat::Binary)) - .Build() - .Settings().Build() - .Format() - .Value(ToString((ui32)IDataProvider::EResultFormat::Yson)) - .Build() - .PublicId() - .Value(TString()) - .Build() - .Discard() - .Value("false") - .Build() - .Origin(calcWorldRoot) - .Done().Ptr(); - - auto atomType = ctx.MakeType<TUnitExprType>(); - for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails, - TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard, TResOrPullBase::idx_Settings }) { - delegatedNode->Child(idx)->SetTypeAnn(atomType); - delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); - } - - delegatedNode->SetTypeAnn(atomType); - delegatedNode->SetState(TExprNode::EState::ConstrComplete); - - status = SyncTransform(calcTransfomer ? *calcTransfomer : (*calcProvider.Get())->GetCallableExecutionTransformer(), delegatedNode, ctx); - - for (auto& dataProvider : types.DataSources) { - dataProvider->UndoEvaluationChanges(); - } - - if (status.Level == IGraphTransformer::TStatus::Error) { - return nullptr; - } - - auto yson = delegatedNode->GetResult().Content(); - ysonNode = NYT::NodeFromYsonString(yson); - if (ysonNode.HasKey("FallbackProvider")) { - nextProvider = ysonNode["FallbackProvider"].AsString(); - } - } while (ysonNode.HasKey("FallbackProvider")); - + .Build() + .Settings().Build() + .Format() + .Value(ToString((ui32)IDataProvider::EResultFormat::Yson)) + .Build() + .PublicId() + .Value(TString()) + .Build() + .Discard() + .Value("false") + .Build() + .Origin(calcWorldRoot) + .Done().Ptr(); + + auto atomType = ctx.MakeType<TUnitExprType>(); + for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails, + TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard, TResOrPullBase::idx_Settings }) { + delegatedNode->Child(idx)->SetTypeAnn(atomType); + delegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete); + } + + delegatedNode->SetTypeAnn(atomType); + delegatedNode->SetState(TExprNode::EState::ConstrComplete); + + status = SyncTransform(calcTransfomer ? *calcTransfomer : (*calcProvider.Get())->GetCallableExecutionTransformer(), delegatedNode, ctx); + + for (auto& dataProvider : types.DataSources) { + dataProvider->UndoEvaluationChanges(); + } + + if (status.Level == IGraphTransformer::TStatus::Error) { + return nullptr; + } + + auto yson = delegatedNode->GetResult().Content(); + ysonNode = NYT::NodeFromYsonString(yson); + if (ysonNode.HasKey("FallbackProvider")) { + nextProvider = ysonNode["FallbackProvider"].AsString(); + } + } while (ysonNode.HasKey("FallbackProvider")); + auto dataNode = ysonNode["Data"]; if (isAtomPipeline) { if (isOptionalAtom) { diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.cpp b/ydb/library/yql/core/services/yql_transform_pipeline.cpp index 2451ba1f37..ae425f34c8 100644 --- a/ydb/library/yql/core/services/yql_transform_pipeline.cpp +++ b/ydb/library/yql/core/services/yql_transform_pipeline.cpp @@ -15,7 +15,7 @@ #include <ydb/library/yql/core/yql_opt_rewrite_io.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - + namespace NYql { TTransformationPipeline::TTransformationPipeline(TIntrusivePtr<TTypeAnnotationContext> ctx) diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h index 292f5213ec..3ffbdc041d 100644 --- a/ydb/library/yql/core/yql_data_provider.h +++ b/ydb/library/yql/core/yql_data_provider.h @@ -129,7 +129,7 @@ public: virtual IGraphTransformer& GetPhysicalOptProposalTransformer() = 0; virtual IGraphTransformer& GetPhysicalFinalizingTransformer() = 0; virtual void PostRewriteIO() = 0; - virtual void Reset() = 0; + virtual void Reset() = 0; //-- metadata loading virtual IGraphTransformer& GetLoadTableMetadataTransformer() = 0; diff --git a/ydb/library/yql/core/yql_graph_transformer.cpp b/ydb/library/yql/core/yql_graph_transformer.cpp index eb0a99c01b..26193c4c40 100644 --- a/ydb/library/yql/core/yql_graph_transformer.cpp +++ b/ydb/library/yql/core/yql_graph_transformer.cpp @@ -99,7 +99,7 @@ public: } private: - virtual TStatus HandleStatus(TStatus status) { + virtual TStatus HandleStatus(TStatus status) { if (status.Level == IGraphTransformer::TStatus::Error) { return status; } @@ -133,7 +133,7 @@ private: } } -protected: +protected: TVector<TTransformStage> Stages; const bool UseIssueScopes; const bool DoCheckArguments; @@ -159,75 +159,75 @@ TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const return new TCompositeGraphTransformer(stages, useIssueScopes, /* doCheckArguments = */ false); } -namespace { - -class TChoiceGraphTransformer : public TCompositeGraphTransformer { -public: - TChoiceGraphTransformer( - const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, - const TTransformStage& left, - const TTransformStage& right) - : TCompositeGraphTransformer( - {WrapCondition(condition), left, right}, - /* useIssueScopes = */ false, - /* doCheckArgumentstrue = */ true) - { } - -private: - void Rewind() override { - Condition.Clear(); - TCompositeGraphTransformer::Rewind(); - } - - TStatus HandleStatus(TStatus status) override { - if (status.Level == IGraphTransformer::TStatus::Error) { - return status; - } - - if (status.HasRestart) { - // ignore Async status in this case - Index = 0; - status = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); - } else if (status.Level == IGraphTransformer::TStatus::Ok) { - status = IGraphTransformer::TStatus::Repeat; - YQL_ENSURE(!Condition.Empty(), "Condition must be set"); - if (Index == 0 && *Condition) { - Index = 1; // left - } else if (Index == 0) { - Index = 2; // right - } else { - Index = 3; // end - } - } - - return status; - } - - TTransformStage WrapCondition(const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition) - { - auto transformer = CreateFunctorTransformer([this, condition](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - output = input; - if (Condition.Empty()) { - Condition = condition(input, ctx); - } - return TStatus::Ok; - }); - - return TTransformStage(transformer, "Condition", TIssuesIds::DEFAULT_ERROR); - } - - TMaybe<bool> Condition; -}; - -} // namespace - -TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( - const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, - const TTransformStage& left, const TTransformStage& right) -{ - return new TChoiceGraphTransformer(condition, left, right); -} - +namespace { + +class TChoiceGraphTransformer : public TCompositeGraphTransformer { +public: + TChoiceGraphTransformer( + const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, + const TTransformStage& left, + const TTransformStage& right) + : TCompositeGraphTransformer( + {WrapCondition(condition), left, right}, + /* useIssueScopes = */ false, + /* doCheckArgumentstrue = */ true) + { } + +private: + void Rewind() override { + Condition.Clear(); + TCompositeGraphTransformer::Rewind(); + } + + TStatus HandleStatus(TStatus status) override { + if (status.Level == IGraphTransformer::TStatus::Error) { + return status; + } + + if (status.HasRestart) { + // ignore Async status in this case + Index = 0; + status = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); + } else if (status.Level == IGraphTransformer::TStatus::Ok) { + status = IGraphTransformer::TStatus::Repeat; + YQL_ENSURE(!Condition.Empty(), "Condition must be set"); + if (Index == 0 && *Condition) { + Index = 1; // left + } else if (Index == 0) { + Index = 2; // right + } else { + Index = 3; // end + } + } + + return status; + } + + TTransformStage WrapCondition(const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition) + { + auto transformer = CreateFunctorTransformer([this, condition](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + output = input; + if (Condition.Empty()) { + Condition = condition(input, ctx); + } + return TStatus::Ok; + }); + + return TTransformStage(transformer, "Condition", TIssuesIds::DEFAULT_ERROR); + } + + TMaybe<bool> Condition; +}; + +} // namespace + +TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( + const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, + const TTransformStage& left, const TTransformStage& right) +{ + return new TChoiceGraphTransformer(condition, left, right); +} + IGraphTransformer::TStatus SyncTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx) { try { for (; ctx.RepeatTransformCounter < ctx.RepeatTransformLimit; ++ctx.RepeatTransformCounter) { diff --git a/ydb/library/yql/core/yql_graph_transformer.h b/ydb/library/yql/core/yql_graph_transformer.h index 872945db07..60d87727dd 100644 --- a/ydb/library/yql/core/yql_graph_transformer.h +++ b/ydb/library/yql/core/yql_graph_transformer.h @@ -220,11 +220,11 @@ private: TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformer(const TVector<TTransformStage>& stages, bool useIssueScopes); TAutoPtr<IGraphTransformer> CreateCompositeGraphTransformerWithNoArgChecks(const TVector<TTransformStage>& stages, bool useIssueScopes); -TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( - const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, - const TTransformStage& left, - const TTransformStage& right); - +TAutoPtr<IGraphTransformer> CreateChoiceGraphTransformer( + const std::function<bool(const TExprNode::TPtr& input, TExprContext& ctx)>& condition, + const TTransformStage& left, + const TTransformStage& right); + IGraphTransformer::TStatus SyncTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx); IGraphTransformer::TStatus InstantTransform(IGraphTransformer& transformer, TExprNode::TPtr& root, TExprContext& ctx, bool breakOnRestart = false); diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h index 640bb44c0c..1a1f1fb476 100644 --- a/ydb/library/yql/core/yql_type_annotation.h +++ b/ydb/library/yql/core/yql_type_annotation.h @@ -188,7 +188,7 @@ struct TTypeAnnotationContext: public TThrRefBase { TVector<TString> AvailablePureResultDataSources; TString FullResultDataSink; TUserDataStorage::TPtr UserDataStorage; - TUserDataTable UserDataStorageCrutches; + TUserDataTable UserDataStorageCrutches; TYqlOperationOptions OperationOptions; TVector<TCredentialTablePtr> Credentials; TUserCredentials UserCredentials; @@ -215,9 +215,9 @@ struct TTypeAnnotationContext: public TThrRefBase { UdfTypeCache; // (name,typecfg,type)->(type,run config type,new user type) bool UseTableMetaFromGraph = false; bool DiscoveryMode = false; - bool ForceDq = false; - bool DqCaptured = false; // TODO: Add before/after recapture transformers - TString DqFallbackPolicy = ""; + bool ForceDq = false; + bool DqCaptured = false; // TODO: Add before/after recapture transformers + TString DqFallbackPolicy = ""; bool StrictTableProps = true; bool JsonQueryReturnsJsonDocument = false; ui32 FolderSubDirsLimit = 1000; diff --git a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp index 3e7d769a3b..6f5f46abe6 100644 --- a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.cpp @@ -1,416 +1,416 @@ -#include "dq_compute_actor.h" -#include "dq_async_compute_actor.h" - -#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> - -#include <ydb/library/yql/dq/common/dq_common.h> - -namespace NYql { -namespace NDq { - -using namespace NActors; - -namespace { - -bool IsDebugLogEnabled(const TActorSystem* actorSystem) { - auto* settings = actorSystem->LoggerSettings(); - return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER); -} - -} // anonymous namespace - -class TDqAsyncComputeActor : public TDqComputeActorBase<TDqAsyncComputeActor> - , public NTaskRunnerActor::ITaskRunnerActor::ICallbacks -{ - - using TBase = TDqComputeActorBase<TDqAsyncComputeActor>; - -public: - static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR"; - - TDqAsyncComputeActor(const TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, - IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, - const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, - const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) - : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits) - , TaskRunnerActorFactory(taskRunnerActorFactory) - {} - - void DoBootstrap() { - const TActorSystem* actorSystem = TlsActivationContext->ActorSystem(); - - TLogFunc logger; - if (IsDebugLogEnabled(actorSystem)) { - logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) { - LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId - << ", task: " << taskId << ": " << message); - }; - } - - NActors::IActor* actor; - std::tie(TaskRunnerActor, actor) = TaskRunnerActorFactory->Create( - this, TStringBuilder() << this->GetTxId()); - TaskRunnerActorId = this->RegisterWithSameMailbox(actor); - - TDqTaskRunnerMemoryLimits limits; - limits.ChannelBufferSize = MemoryLimits.ChannelBufferSize; - limits.OutputChunkMaxSize = GetDqExecutionSettings().FlowControl.MaxOutputChunkSize; - - this->Become(&TDqAsyncComputeActor::StateFuncBase); - - // TODO: - std::shared_ptr<IDqTaskRunnerExecutionContext> execCtx = std::shared_ptr<IDqTaskRunnerExecutionContext>(new TDqTaskRunnerExecutionContext()); - - this->Send(TaskRunnerActorId, - new NTaskRunnerActor::TEvTaskRunnerCreate( - GetTask(), limits, execCtx)); - } - - void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) { - } - -private: - STFUNC(StateFuncBase) { - switch (ev->GetTypeRewrite()) { - HFunc(NTaskRunnerActor::TEvTaskRunFinished, OnRunFinished); - HFunc(NTaskRunnerActor::TEvSourcePushFinished, OnSourcePushFinished); - HFunc(NTaskRunnerActor::TEvChannelPopFinished, OnPopFinished); - HFunc(NTaskRunnerActor::TEvTaskRunnerCreateFinished, OnTaskRunnerCreatead); - HFunc(NTaskRunnerActor::TEvContinueRun, OnContinueRun); // push finished - default: - TDqComputeActorBase<TDqAsyncComputeActor>::StateFuncBase(ev, ctx); - }; - }; - - void DrainOutputChannel(TOutputChannelInfo& outputChannel, const TDqComputeActorChannels::TPeerState& peerState) override { - YQL_ENSURE(!outputChannel.Finished || Checkpoints); - - if (outputChannel.PopStarted) { - return; - } - - const bool wasFinished = outputChannel.Finished; - auto channelId = outputChannel.ChannelId; - - const ui32 allowedOvercommit = AllowedChannelsOvercommit(); - - const i64 toSend = peerState.PeerFreeSpace + allowedOvercommit - peerState.InFlightBytes; - - CA_LOG_D("About to drain channelId: " << channelId - << ", hasPeer: " << outputChannel.HasPeer - << ", peerFreeSpace: " << peerState.PeerFreeSpace - << ", inFlightBytes: " << peerState.InFlightBytes - << ", inFlightRows: " << peerState.InFlightRows - << ", inFlightCount: " << peerState.InFlightCount - << ", allowedOvercommit: " << allowedOvercommit - << ", toSend: " << toSend -// << ", finished: " << outputChannel.Channel->IsFinished()); - ); - - outputChannel.PopStarted = true; - ProcessOutputsState.Inflight ++; - if (toSend <= 0) { - if (Y_UNLIKELY(outputChannel.Stats)) { - outputChannel.Stats->BlockedByCapacity++; - } - this->Send(this->SelfId(), new NTaskRunnerActor::TEvChannelPopFinished(channelId)); - return; - } - - this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvPop(channelId, wasFinished, toSend)); - } - - void DrainSink(ui64 outputIndex, TSinkInfo& sinkInfo) override { - if (sinkInfo.Finished && !Checkpoints) { - return; - } - - if (sinkInfo.PopStarted) { - return; - } - - Y_VERIFY(sinkInfo.SinkActor); - Y_VERIFY(sinkInfo.Actor); - - const ui32 allowedOvercommit = AllowedChannelsOvercommit(); - const i64 sinkActorFreeSpaceBeforeSend = sinkInfo.SinkActor->GetFreeSpace(); - - i64 toSend = sinkActorFreeSpaceBeforeSend + allowedOvercommit; - CA_LOG_D("About to drain sink " << outputIndex - << ". FreeSpace: " << sinkActorFreeSpaceBeforeSend - << ", allowedOvercommit: " << allowedOvercommit - << ", toSend: " << toSend - //<< ", finished: " << sinkInfo.Sink->IsFinished()); - ); - - sinkInfo.PopStarted = true; - ProcessOutputsState.Inflight ++; - sinkInfo.SinkActorFreeSpaceBeforeSend = sinkActorFreeSpaceBeforeSend; - } - - void SourcePush(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, TSourceInfo& source, i64 space, bool finished) override { - if (space <= 0) { - return; - } - - ProcessSourcesState.Inflight ++; - source.PushStarted = true; - source.Finished = finished; - TaskRunnerActor->SourcePush(Cookie++, source.Index, std::move(batch), space, finished); - } - - void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) override { - TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelData.GetChannelId()); - YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelData.GetChannelId()); - - auto finished = channelData.GetFinished(); - - auto ev = (channelData.GetData().GetRows()) - ? MakeHolder<NTaskRunnerActor::TEvPush>( - channelData.GetChannelId(), - std::move(*channelData.MutableData()), - finished, - /*askFreeSpace = */ true) - : MakeHolder<NTaskRunnerActor::TEvPush>( - channelData.GetChannelId(), finished, /*askFreeSpace = */ true); - - this->Send(TaskRunnerActorId, ev.Release(), 0, Cookie); - - if (channelData.HasCheckpoint()) { - Y_VERIFY(inputChannel->CheckpointingMode != NDqProto::CHECKPOINTING_MODE_DISABLED); - Y_VERIFY(Checkpoints); - const auto& checkpoint = channelData.GetCheckpoint(); - inputChannel->Pause(checkpoint); - Checkpoints->RegisterCheckpoint(checkpoint, channelData.GetChannelId()); - } - - TakeInputChannelDataRequests[Cookie++] = TTakeInputChannelData{ack, channelData.GetChannelId()}; - } - - void PassAway() override { - if (TaskRunnerActor) { - TaskRunnerActor->PassAway(); - } - NActors::IActor::PassAway(); - } - - void DoExecuteImpl() override { - PollSourceActors(); - if (ProcessSourcesState.Inflight == 0) { - this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvContinueRun()); - } - } - - bool SayHelloOnBootstrap() override { - return false; - } - - i64 GetInputChannelFreeSpace(ui64 channelId) const override { - const TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelId); - YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelId); - - return inputChannel->FreeSpace; - } - - i64 SourceFreeSpace(TSourceInfo& source) override { - return source.FreeSpace; - } - - TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator() override { - return TypeEnv->BindAllocator(); - } - - std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> MaybeBindAllocator() override { - std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard; - if (TypeEnv) { - guard.emplace(TypeEnv->BindAllocator()); - } - return guard; - } - - void OnTaskRunnerCreatead(NTaskRunnerActor::TEvTaskRunnerCreateFinished::TPtr& ev, const NActors::TActorContext& ) { - const auto& secureParams = ev->Get()->SecureParams; - const auto& taskParams = ev->Get()->TaskParams; - const auto& typeEnv = ev->Get()->TypeEnv; - const auto& holderFactory = ev->Get()->HolderFactory; - - TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); - FillChannelMaps(holderFactory, typeEnv, secureParams, taskParams); - - { - // say "Hello" to executer - auto ev = MakeHolder<TEvDqCompute::TEvState>(); - ev->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING); - ev->Record.SetTaskId(Task.GetId()); - - this->Send(ExecuterId, ev.Release(), NActors::IEventHandle::FlagTrackDelivery); - } - - ContinueExecute(); - } - - void OnRunFinished(NTaskRunnerActor::TEvTaskRunFinished::TPtr& ev, const NActors::TActorContext& ) { - auto sourcesState = GetSourcesState(); - auto status = ev->Get()->RunStatus; - - CA_LOG_D("Resume execution, run status: " << status); - - for (const auto& [channelId, freeSpace] : ev->Get()->InputChannelFreeSpace) { - auto it = InputChannelsMap.find(channelId); - if (it != InputChannelsMap.end()) { - it->second.FreeSpace = freeSpace; - } - } - - if (status != ERunStatus::Finished) { - PollSources(std::move(sourcesState)); - } - - if ((status == ERunStatus::PendingInput || status == ERunStatus::Finished) && Checkpoints && Checkpoints->HasPendingCheckpoint() && !Checkpoints->ComputeActorStateSaved() && ReadyToCheckpoint()) { - Checkpoints->DoCheckpoint(); - } - - ProcessOutputsImpl(status); - } - - void OnSourcePushFinished(NTaskRunnerActor::TEvSourcePushFinished::TPtr& ev, const NActors::TActorContext& ) { - auto it = SourcesMap.find(ev->Get()->Index); - Y_VERIFY(it != SourcesMap.end()); - auto& source = it->second; - source.PushStarted = false; - // source.FreeSpace = ev->Get()->FreeSpace; TODO:XXX get freespace on run - ProcessSourcesState.Inflight--; - if (ProcessSourcesState.Inflight == 0) { - this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvContinueRun()); - } - } - - void OnPopFinished(NTaskRunnerActor::TEvChannelPopFinished::TPtr& ev, const NActors::TActorContext&) { - auto channelId = ev->Get()->ChannelId; - auto finished = ev->Get()->Finished; - auto dataWasSent = ev->Get()->Changed; - auto it = OutputChannelsMap.find(channelId); - Y_VERIFY(it != OutputChannelsMap.end()); - - TOutputChannelInfo& outputChannel = it->second; - outputChannel.Finished = finished; - if (finished) { - FinishedOutputChannels.insert(channelId); - } - - outputChannel.PopStarted = false; - ProcessOutputsState.Inflight --; - ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; - - for (ui32 i = 0; i < ev->Get()->Data.size(); i++) { - auto& chunk = ev->Get()->Data[i]; - NDqProto::TChannelData channelData; - channelData.SetChannelId(channelId); - // set finished only for last chunk - channelData.SetFinished(finished && i==ev->Get()->Data.size()-1); - channelData.MutableData()->Swap(&chunk); - Channels->SendChannelData(std::move(channelData)); - } - if (ev->Get()->Data.empty() && dataWasSent) { - NDqProto::TChannelData channelData; - channelData.SetChannelId(channelId); - channelData.SetFinished(finished); - Channels->SendChannelData(std::move(channelData)); - } - - ProcessOutputsState.DataWasSent |= dataWasSent; - - ProcessOutputsState.AllOutputsFinished = - FinishedOutputChannels.size() == OutputChannelsMap.size() && - FinishedSinks.size() == SinksMap.size(); - - CheckRunStatus(); - } - - void OnContinueRun(NTaskRunnerActor::TEvContinueRun::TPtr& ev, const NActors::TActorContext& ) { - auto it = TakeInputChannelDataRequests.find(ev->Cookie); - YQL_ENSURE(it != TakeInputChannelDataRequests.end()); - - if (it->second.Ack) { - TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(it->second.ChannelId); - Channels->SendChannelDataAck(it->second.ChannelId, inputChannel->FreeSpace); - } - - ResumeExecution(); - } - - void SinkSend(ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - TMaybe<NDqProto::TCheckpoint>&& checkpoint, - i64 size, - i64 checkpointSize, - bool finished, - bool changed) override - { - auto outputIndex = index; - auto dataWasSent = finished || changed; - auto dataSize = size; - auto it = SinksMap.find(outputIndex); - Y_VERIFY(it != SinksMap.end()); - - TSinkInfo& sinkInfo = it->second; - sinkInfo.Finished = finished; - if (finished) { - FinishedSinks.insert(outputIndex); - } - if (checkpoint) { - CA_LOG_I("Resume inputs"); - ResumeInputs(); - } - - sinkInfo.PopStarted = false; - ProcessOutputsState.Inflight --; - ProcessOutputsState.HasDataToSend |= !sinkInfo.Finished; - - auto guard = BindAllocator(); - sinkInfo.SinkActor->SendData(std::move(batch), size, std::move(checkpoint), finished); - CA_LOG_D("sink " << outputIndex << ": sent " << dataSize << " bytes of data and " << checkpointSize << " bytes of checkpoint barrier"); - - CA_LOG_D("Drain sink " << outputIndex - << ". Free space decreased: " << (sinkInfo.SinkActorFreeSpaceBeforeSend - sinkInfo.SinkActor->GetFreeSpace()) - << ", sent data from buffer: " << dataSize); - - ProcessOutputsState.DataWasSent |= dataWasSent; - ProcessOutputsState.AllOutputsFinished = - FinishedOutputChannels.size() == OutputChannelsMap.size() && - FinishedSinks.size() == SinksMap.size(); - CheckRunStatus(); - } - - NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv; - NTaskRunnerActor::ITaskRunnerActor* TaskRunnerActor; - NActors::TActorId TaskRunnerActorId; - NTaskRunnerActor::ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; - - THashSet<ui64> FinishedOutputChannels; - THashSet<ui64> FinishedSinks; - struct TProcessSourcesState { - int Inflight = 0; - }; - TProcessSourcesState ProcessSourcesState; - - struct TTakeInputChannelData { - bool Ack; - ui64 ChannelId; - }; - THashMap<ui64, TTakeInputChannelData> TakeInputChannelDataRequests; - ui64 Cookie = 0; -}; - - -IActor* CreateDqAsyncComputeActor(const TActorId& executerId, const TTxId& txId, NYql::NDqProto::TDqTask&& task, - IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, - const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, - const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) -{ - return new TDqAsyncComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory), - std::move(sinkActorFactory), settings, memoryLimits, taskRunnerActorFactory); -} - -} // namespace NDq -} // namespace NYql +#include "dq_compute_actor.h" +#include "dq_async_compute_actor.h" + +#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> + +#include <ydb/library/yql/dq/common/dq_common.h> + +namespace NYql { +namespace NDq { + +using namespace NActors; + +namespace { + +bool IsDebugLogEnabled(const TActorSystem* actorSystem) { + auto* settings = actorSystem->LoggerSettings(); + return settings && settings->Satisfies(NActors::NLog::EPriority::PRI_DEBUG, NKikimrServices::KQP_TASKS_RUNNER); +} + +} // anonymous namespace + +class TDqAsyncComputeActor : public TDqComputeActorBase<TDqAsyncComputeActor> + , public NTaskRunnerActor::ITaskRunnerActor::ICallbacks +{ + + using TBase = TDqComputeActorBase<TDqAsyncComputeActor>; + +public: + static constexpr char ActorName[] = "DQ_COMPUTE_ACTOR"; + + TDqAsyncComputeActor(const TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, + IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, + const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, + const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) + : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits) + , TaskRunnerActorFactory(taskRunnerActorFactory) + {} + + void DoBootstrap() { + const TActorSystem* actorSystem = TlsActivationContext->ActorSystem(); + + TLogFunc logger; + if (IsDebugLogEnabled(actorSystem)) { + logger = [actorSystem, txId = this->GetTxId(), taskId = GetTask().GetId()] (const TString& message) { + LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId + << ", task: " << taskId << ": " << message); + }; + } + + NActors::IActor* actor; + std::tie(TaskRunnerActor, actor) = TaskRunnerActorFactory->Create( + this, TStringBuilder() << this->GetTxId()); + TaskRunnerActorId = this->RegisterWithSameMailbox(actor); + + TDqTaskRunnerMemoryLimits limits; + limits.ChannelBufferSize = MemoryLimits.ChannelBufferSize; + limits.OutputChunkMaxSize = GetDqExecutionSettings().FlowControl.MaxOutputChunkSize; + + this->Become(&TDqAsyncComputeActor::StateFuncBase); + + // TODO: + std::shared_ptr<IDqTaskRunnerExecutionContext> execCtx = std::shared_ptr<IDqTaskRunnerExecutionContext>(new TDqTaskRunnerExecutionContext()); + + this->Send(TaskRunnerActorId, + new NTaskRunnerActor::TEvTaskRunnerCreate( + GetTask(), limits, execCtx)); + } + + void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) { + } + +private: + STFUNC(StateFuncBase) { + switch (ev->GetTypeRewrite()) { + HFunc(NTaskRunnerActor::TEvTaskRunFinished, OnRunFinished); + HFunc(NTaskRunnerActor::TEvSourcePushFinished, OnSourcePushFinished); + HFunc(NTaskRunnerActor::TEvChannelPopFinished, OnPopFinished); + HFunc(NTaskRunnerActor::TEvTaskRunnerCreateFinished, OnTaskRunnerCreatead); + HFunc(NTaskRunnerActor::TEvContinueRun, OnContinueRun); // push finished + default: + TDqComputeActorBase<TDqAsyncComputeActor>::StateFuncBase(ev, ctx); + }; + }; + + void DrainOutputChannel(TOutputChannelInfo& outputChannel, const TDqComputeActorChannels::TPeerState& peerState) override { + YQL_ENSURE(!outputChannel.Finished || Checkpoints); + + if (outputChannel.PopStarted) { + return; + } + + const bool wasFinished = outputChannel.Finished; + auto channelId = outputChannel.ChannelId; + + const ui32 allowedOvercommit = AllowedChannelsOvercommit(); + + const i64 toSend = peerState.PeerFreeSpace + allowedOvercommit - peerState.InFlightBytes; + + CA_LOG_D("About to drain channelId: " << channelId + << ", hasPeer: " << outputChannel.HasPeer + << ", peerFreeSpace: " << peerState.PeerFreeSpace + << ", inFlightBytes: " << peerState.InFlightBytes + << ", inFlightRows: " << peerState.InFlightRows + << ", inFlightCount: " << peerState.InFlightCount + << ", allowedOvercommit: " << allowedOvercommit + << ", toSend: " << toSend +// << ", finished: " << outputChannel.Channel->IsFinished()); + ); + + outputChannel.PopStarted = true; + ProcessOutputsState.Inflight ++; + if (toSend <= 0) { + if (Y_UNLIKELY(outputChannel.Stats)) { + outputChannel.Stats->BlockedByCapacity++; + } + this->Send(this->SelfId(), new NTaskRunnerActor::TEvChannelPopFinished(channelId)); + return; + } + + this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvPop(channelId, wasFinished, toSend)); + } + + void DrainSink(ui64 outputIndex, TSinkInfo& sinkInfo) override { + if (sinkInfo.Finished && !Checkpoints) { + return; + } + + if (sinkInfo.PopStarted) { + return; + } + + Y_VERIFY(sinkInfo.SinkActor); + Y_VERIFY(sinkInfo.Actor); + + const ui32 allowedOvercommit = AllowedChannelsOvercommit(); + const i64 sinkActorFreeSpaceBeforeSend = sinkInfo.SinkActor->GetFreeSpace(); + + i64 toSend = sinkActorFreeSpaceBeforeSend + allowedOvercommit; + CA_LOG_D("About to drain sink " << outputIndex + << ". FreeSpace: " << sinkActorFreeSpaceBeforeSend + << ", allowedOvercommit: " << allowedOvercommit + << ", toSend: " << toSend + //<< ", finished: " << sinkInfo.Sink->IsFinished()); + ); + + sinkInfo.PopStarted = true; + ProcessOutputsState.Inflight ++; + sinkInfo.SinkActorFreeSpaceBeforeSend = sinkActorFreeSpaceBeforeSend; + } + + void SourcePush(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, TSourceInfo& source, i64 space, bool finished) override { + if (space <= 0) { + return; + } + + ProcessSourcesState.Inflight ++; + source.PushStarted = true; + source.Finished = finished; + TaskRunnerActor->SourcePush(Cookie++, source.Index, std::move(batch), space, finished); + } + + void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) override { + TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelData.GetChannelId()); + YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelData.GetChannelId()); + + auto finished = channelData.GetFinished(); + + auto ev = (channelData.GetData().GetRows()) + ? MakeHolder<NTaskRunnerActor::TEvPush>( + channelData.GetChannelId(), + std::move(*channelData.MutableData()), + finished, + /*askFreeSpace = */ true) + : MakeHolder<NTaskRunnerActor::TEvPush>( + channelData.GetChannelId(), finished, /*askFreeSpace = */ true); + + this->Send(TaskRunnerActorId, ev.Release(), 0, Cookie); + + if (channelData.HasCheckpoint()) { + Y_VERIFY(inputChannel->CheckpointingMode != NDqProto::CHECKPOINTING_MODE_DISABLED); + Y_VERIFY(Checkpoints); + const auto& checkpoint = channelData.GetCheckpoint(); + inputChannel->Pause(checkpoint); + Checkpoints->RegisterCheckpoint(checkpoint, channelData.GetChannelId()); + } + + TakeInputChannelDataRequests[Cookie++] = TTakeInputChannelData{ack, channelData.GetChannelId()}; + } + + void PassAway() override { + if (TaskRunnerActor) { + TaskRunnerActor->PassAway(); + } + NActors::IActor::PassAway(); + } + + void DoExecuteImpl() override { + PollSourceActors(); + if (ProcessSourcesState.Inflight == 0) { + this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvContinueRun()); + } + } + + bool SayHelloOnBootstrap() override { + return false; + } + + i64 GetInputChannelFreeSpace(ui64 channelId) const override { + const TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelId); + YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelId); + + return inputChannel->FreeSpace; + } + + i64 SourceFreeSpace(TSourceInfo& source) override { + return source.FreeSpace; + } + + TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator() override { + return TypeEnv->BindAllocator(); + } + + std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> MaybeBindAllocator() override { + std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard; + if (TypeEnv) { + guard.emplace(TypeEnv->BindAllocator()); + } + return guard; + } + + void OnTaskRunnerCreatead(NTaskRunnerActor::TEvTaskRunnerCreateFinished::TPtr& ev, const NActors::TActorContext& ) { + const auto& secureParams = ev->Get()->SecureParams; + const auto& taskParams = ev->Get()->TaskParams; + const auto& typeEnv = ev->Get()->TypeEnv; + const auto& holderFactory = ev->Get()->HolderFactory; + + TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); + FillChannelMaps(holderFactory, typeEnv, secureParams, taskParams); + + { + // say "Hello" to executer + auto ev = MakeHolder<TEvDqCompute::TEvState>(); + ev->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING); + ev->Record.SetTaskId(Task.GetId()); + + this->Send(ExecuterId, ev.Release(), NActors::IEventHandle::FlagTrackDelivery); + } + + ContinueExecute(); + } + + void OnRunFinished(NTaskRunnerActor::TEvTaskRunFinished::TPtr& ev, const NActors::TActorContext& ) { + auto sourcesState = GetSourcesState(); + auto status = ev->Get()->RunStatus; + + CA_LOG_D("Resume execution, run status: " << status); + + for (const auto& [channelId, freeSpace] : ev->Get()->InputChannelFreeSpace) { + auto it = InputChannelsMap.find(channelId); + if (it != InputChannelsMap.end()) { + it->second.FreeSpace = freeSpace; + } + } + + if (status != ERunStatus::Finished) { + PollSources(std::move(sourcesState)); + } + + if ((status == ERunStatus::PendingInput || status == ERunStatus::Finished) && Checkpoints && Checkpoints->HasPendingCheckpoint() && !Checkpoints->ComputeActorStateSaved() && ReadyToCheckpoint()) { + Checkpoints->DoCheckpoint(); + } + + ProcessOutputsImpl(status); + } + + void OnSourcePushFinished(NTaskRunnerActor::TEvSourcePushFinished::TPtr& ev, const NActors::TActorContext& ) { + auto it = SourcesMap.find(ev->Get()->Index); + Y_VERIFY(it != SourcesMap.end()); + auto& source = it->second; + source.PushStarted = false; + // source.FreeSpace = ev->Get()->FreeSpace; TODO:XXX get freespace on run + ProcessSourcesState.Inflight--; + if (ProcessSourcesState.Inflight == 0) { + this->Send(TaskRunnerActorId, new NTaskRunnerActor::TEvContinueRun()); + } + } + + void OnPopFinished(NTaskRunnerActor::TEvChannelPopFinished::TPtr& ev, const NActors::TActorContext&) { + auto channelId = ev->Get()->ChannelId; + auto finished = ev->Get()->Finished; + auto dataWasSent = ev->Get()->Changed; + auto it = OutputChannelsMap.find(channelId); + Y_VERIFY(it != OutputChannelsMap.end()); + + TOutputChannelInfo& outputChannel = it->second; + outputChannel.Finished = finished; + if (finished) { + FinishedOutputChannels.insert(channelId); + } + + outputChannel.PopStarted = false; + ProcessOutputsState.Inflight --; + ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; + + for (ui32 i = 0; i < ev->Get()->Data.size(); i++) { + auto& chunk = ev->Get()->Data[i]; + NDqProto::TChannelData channelData; + channelData.SetChannelId(channelId); + // set finished only for last chunk + channelData.SetFinished(finished && i==ev->Get()->Data.size()-1); + channelData.MutableData()->Swap(&chunk); + Channels->SendChannelData(std::move(channelData)); + } + if (ev->Get()->Data.empty() && dataWasSent) { + NDqProto::TChannelData channelData; + channelData.SetChannelId(channelId); + channelData.SetFinished(finished); + Channels->SendChannelData(std::move(channelData)); + } + + ProcessOutputsState.DataWasSent |= dataWasSent; + + ProcessOutputsState.AllOutputsFinished = + FinishedOutputChannels.size() == OutputChannelsMap.size() && + FinishedSinks.size() == SinksMap.size(); + + CheckRunStatus(); + } + + void OnContinueRun(NTaskRunnerActor::TEvContinueRun::TPtr& ev, const NActors::TActorContext& ) { + auto it = TakeInputChannelDataRequests.find(ev->Cookie); + YQL_ENSURE(it != TakeInputChannelDataRequests.end()); + + if (it->second.Ack) { + TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(it->second.ChannelId); + Channels->SendChannelDataAck(it->second.ChannelId, inputChannel->FreeSpace); + } + + ResumeExecution(); + } + + void SinkSend(ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + TMaybe<NDqProto::TCheckpoint>&& checkpoint, + i64 size, + i64 checkpointSize, + bool finished, + bool changed) override + { + auto outputIndex = index; + auto dataWasSent = finished || changed; + auto dataSize = size; + auto it = SinksMap.find(outputIndex); + Y_VERIFY(it != SinksMap.end()); + + TSinkInfo& sinkInfo = it->second; + sinkInfo.Finished = finished; + if (finished) { + FinishedSinks.insert(outputIndex); + } + if (checkpoint) { + CA_LOG_I("Resume inputs"); + ResumeInputs(); + } + + sinkInfo.PopStarted = false; + ProcessOutputsState.Inflight --; + ProcessOutputsState.HasDataToSend |= !sinkInfo.Finished; + + auto guard = BindAllocator(); + sinkInfo.SinkActor->SendData(std::move(batch), size, std::move(checkpoint), finished); + CA_LOG_D("sink " << outputIndex << ": sent " << dataSize << " bytes of data and " << checkpointSize << " bytes of checkpoint barrier"); + + CA_LOG_D("Drain sink " << outputIndex + << ". Free space decreased: " << (sinkInfo.SinkActorFreeSpaceBeforeSend - sinkInfo.SinkActor->GetFreeSpace()) + << ", sent data from buffer: " << dataSize); + + ProcessOutputsState.DataWasSent |= dataWasSent; + ProcessOutputsState.AllOutputsFinished = + FinishedOutputChannels.size() == OutputChannelsMap.size() && + FinishedSinks.size() == SinksMap.size(); + CheckRunStatus(); + } + + NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv; + NTaskRunnerActor::ITaskRunnerActor* TaskRunnerActor; + NActors::TActorId TaskRunnerActorId; + NTaskRunnerActor::ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; + + THashSet<ui64> FinishedOutputChannels; + THashSet<ui64> FinishedSinks; + struct TProcessSourcesState { + int Inflight = 0; + }; + TProcessSourcesState ProcessSourcesState; + + struct TTakeInputChannelData { + bool Ack; + ui64 ChannelId; + }; + THashMap<ui64, TTakeInputChannelData> TakeInputChannelDataRequests; + ui64 Cookie = 0; +}; + + +IActor* CreateDqAsyncComputeActor(const TActorId& executerId, const TTxId& txId, NYql::NDqProto::TDqTask&& task, + IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, + const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, + const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) +{ + return new TDqAsyncComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory), + std::move(sinkActorFactory), settings, memoryLimits, taskRunnerActorFactory); +} + +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h index ee477912d4..71c918f04c 100644 --- a/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h @@ -1,27 +1,27 @@ -#pragma once - -#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> -#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> -#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> -#include <ydb/library/yql/dq/actors/dq_events_ids.h> -#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> -#include <ydb/library/yql/dq/actors/task_runner/task_runner_actor.h> -#include <ydb/library/yql/dq/common/dq_common.h> -#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> -#include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> -#include <ydb/library/yql/dq/runtime/dq_transport.h> - -#include <library/cpp/actors/core/actor_bootstrapped.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/log.h> - -namespace NYql { -namespace NDq { - -NActors::IActor* CreateDqAsyncComputeActor(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, - IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, - const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, - const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory); - -} // namespace NDq -} // namespace NYql +#pragma once + +#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sources.h> +#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_sinks.h> +#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> +#include <ydb/library/yql/dq/actors/dq_events_ids.h> +#include <ydb/library/yql/dq/actors/protos/dq_events.pb.h> +#include <ydb/library/yql/dq/actors/task_runner/task_runner_actor.h> +#include <ydb/library/yql/dq/common/dq_common.h> +#include <ydb/library/yql/dq/proto/dq_checkpoint.pb.h> +#include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> +#include <ydb/library/yql/dq/runtime/dq_transport.h> + +#include <library/cpp/actors/core/actor_bootstrapped.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +namespace NYql { +namespace NDq { + +NActors::IActor* CreateDqAsyncComputeActor(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, + IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, + const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, + const NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory); + +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp index 3ce8869fea..c2f6c45e2b 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.cpp @@ -1,12 +1,12 @@ -#include "dq_compute_actor_impl.h" -#include "dq_compute_actor.h" +#include "dq_compute_actor_impl.h" +#include "dq_compute_actor.h" #include <ydb/library/yql/dq/common/dq_common.h> namespace NYql { namespace NDq { -using namespace NActors; +using namespace NActors; namespace { TDqExecutionSettings ExecutionSettings; @@ -37,8 +37,8 @@ public: const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, const TTaskRunnerFactory& taskRunnerFactory) : TBase(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits) - , TaskRunnerFactory(taskRunnerFactory) - {} + , TaskRunnerFactory(taskRunnerFactory) + {} void DoBootstrap() { const TActorSystem* actorSystem = TlsActivationContext->ActorSystem(); @@ -57,22 +57,22 @@ public: ContinueExecute(); } - + void FillExtraStats(NDqProto::TDqComputeActorStats* /* dst */, bool /* last */) { } -private: - const TTaskRunnerFactory TaskRunnerFactory; +private: + const TTaskRunnerFactory TaskRunnerFactory; }; -IActor* CreateDqComputeActor(const TActorId& executerId, const TTxId& txId, NYql::NDqProto::TDqTask&& task, +IActor* CreateDqComputeActor(const TActorId& executerId, const TTxId& txId, NYql::NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, const TTaskRunnerFactory& taskRunnerFactory) -{ +{ return new TDqComputeActor(executerId, txId, std::move(task), std::move(sourceActorFactory), std::move(sinkActorFactory), settings, memoryLimits, taskRunnerFactory); -} - +} + } // namespace NDq } // namespace NYql diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h index 93c9bccdb3..84eeda2edb 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor.h @@ -126,7 +126,7 @@ struct TEvDqCompute { } }; - struct TEvRestoreFromCheckpoint : public NActors::TEventPB<TEvRestoreFromCheckpoint, + struct TEvRestoreFromCheckpoint : public NActors::TEventPB<TEvRestoreFromCheckpoint, NDqProto::TEvRestoreFromCheckpoint, TDqComputeEvents::EvRestoreFromCheckpoint> { TEvRestoreFromCheckpoint() = default; @@ -149,7 +149,7 @@ struct TEvDqCompute { } }; - struct TEvRestoreFromCheckpointResult : public NActors::TEventPB<TEvRestoreFromCheckpointResult, + struct TEvRestoreFromCheckpointResult : public NActors::TEventPB<TEvRestoreFromCheckpointResult, NDqProto::TEvRestoreFromCheckpointResult, TDqComputeEvents::EvRestoreFromCheckpointResult> { using TBaseEventPB = NActors::TEventPB<TEvRestoreFromCheckpointResult, NDqProto::TEvRestoreFromCheckpointResult, TDqComputeEvents::EvRestoreFromCheckpointResult>; @@ -251,14 +251,14 @@ struct TComputeMemoryLimits { using TTaskRunnerFactory = std::function< TIntrusivePtr<IDqTaskRunner>(const NDqProto::TDqTask& task, const TLogFunc& logFunc) >; - + void FillTaskRunnerStats(ui64 taskId, ui32 stageId, const TDqTaskRunnerStats& taskStats, NDqProto::TDqTaskStats* protoTask, bool withProfileStats); -NActors::IActor* CreateDqComputeActor(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, +NActors::IActor* CreateDqComputeActor(const NActors::TActorId& executerId, const TTxId& txId, NDqProto::TDqTask&& task, IDqSourceActorFactory::TPtr sourceActorFactory, IDqSinkActorFactory::TPtr sinkActorFactory, const TComputeRuntimeSettings& settings, const TComputeMemoryLimits& memoryLimits, const TTaskRunnerFactory& taskRunnerFactory); - + } // namespace NDq } // namespace NYql diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp index 06b7b52afe..1a1611ec81 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.cpp @@ -1,4 +1,4 @@ -#include "dq_compute_actor_channels.h" +#include "dq_compute_actor_channels.h" #include <util/string/join.h> @@ -31,7 +31,7 @@ TString InFlightMessagesStr(const TCollection& inFlight) { } // anonymous namespace -TDqComputeActorChannels::TDqComputeActorChannels(TActorId owner, const TTxId& txId, const NDqProto::TDqTask& task, +TDqComputeActorChannels::TDqComputeActorChannels(TActorId owner, const TTxId& txId, const NDqProto::TDqTask& task, bool retryOnUndelivery, NDqProto::EDqStatsMode statsMode, ui64 channelBufferSize, ICallbacks* cbs, ui32 actorActivityType) : TActor(&TDqComputeActorChannels::WorkState, actorActivityType) , Owner(owner) @@ -151,7 +151,7 @@ void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelData::TPtr& ev) inputChannel.Finished = true; } - Cbs->TakeInputChannelData(std::move(*record.MutableChannelData()), !record.GetNoAck()); + Cbs->TakeInputChannelData(std::move(*record.MutableChannelData()), !record.GetNoAck()); } void TDqComputeActorChannels::HandleWork(TEvDqCompute::TEvChannelDataAck::TPtr& ev) { @@ -683,11 +683,11 @@ const TDqComputeActorChannels::TOutputChannelStats* TDqComputeActorChannels::Get return OutCh(channelId).Stats.get(); } -void TDqComputeActorChannels::SendChannelDataAck(i64 channelId, i64 freeSpace) { - TInputChannelState& inputChannel = InCh(channelId); - SendChannelDataAck(inputChannel, freeSpace); -} - +void TDqComputeActorChannels::SendChannelDataAck(i64 channelId, i64 freeSpace) { + TInputChannelState& inputChannel = InCh(channelId); + SendChannelDataAck(inputChannel, freeSpace); +} + void TDqComputeActorChannels::SendChannelDataAck(TInputChannelState& inputChannel, i64 freeSpace) { LOG_D("Sending channel data ack to" << " channelId: " << inputChannel.ChannelId diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h index e1f2b5ee81..648b1d8b56 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_channels.h @@ -1,6 +1,6 @@ #pragma once -#include "dq_compute_actor.h" +#include "dq_compute_actor.h" #include <ydb/core/kqp/kqp.h> @@ -21,7 +21,7 @@ public: struct ICallbacks { virtual i64 GetInputChannelFreeSpace(ui64 channelId) const = 0; - virtual void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) = 0; + virtual void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) = 0; virtual void PeerFinished(ui64 channelId) = 0; virtual void ResumeExecution() = 0; @@ -39,7 +39,7 @@ public: }; public: - TDqComputeActorChannels(NActors::TActorId owner, const TTxId& txId, const NYql::NDqProto::TDqTask& task, bool retryOnUndelivery, + TDqComputeActorChannels(NActors::TActorId owner, const TTxId& txId, const NYql::NDqProto::TDqTask& task, bool retryOnUndelivery, NDqProto::EDqStatsMode statsMode, ui64 channelBufferSize, ICallbacks* cbs, ui32 actorActivityType); private: @@ -71,7 +71,7 @@ public: void SetOutputChannelPeer(ui64 channelId, const NActors::TActorId& peer); bool CanSendChannelData(ui64 channelId); void SendChannelData(NDqProto::TChannelData&& channelData); - void SendChannelDataAck(i64 channelId, i64 freeSpace); + void SendChannelDataAck(i64 channelId, i64 freeSpace); bool PollChannel(ui64 channelId, i64 freeSpace); bool CheckInFlight(const TString& prefix); bool FinishInputChannels(); @@ -175,7 +175,7 @@ private: private: const NActors::TActorId Owner; - const TTxId TxId; + const TTxId TxId; const ui64 TaskId; const bool RetryOnUndelivery; bool SupportCheckpoints = false; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp index 76d70f5c3d..9d943cbca3 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.cpp @@ -24,8 +24,8 @@ namespace NYql::NDq { -using namespace NActors; - +using namespace NActors; + namespace { TString MakeStringForLog(const NDqProto::TCheckpoint& checkpoint) { diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h index 92e0ee597c..27fd851338 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_checkpoints.h @@ -25,10 +25,10 @@ NDqProto::ECheckpointingMode GetTaskCheckpointingMode(const NDqProto::TDqTask& t class TDqComputeActorCheckpoints : public NActors::TActor<TDqComputeActorCheckpoints> { struct TCheckpointCoordinatorId { - NActors::TActorId ActorId; + NActors::TActorId ActorId; ui64 Generation; - TCheckpointCoordinatorId(NActors::TActorId actorId, ui64 generation) + TCheckpointCoordinatorId(NActors::TActorId actorId, ui64 generation) : ActorId(actorId) , Generation(generation) { } @@ -122,7 +122,7 @@ private: const NDqProto::TDqTask Task; const bool IngressTask; - const NActors::TActorId CheckpointStorage; + const NActors::TActorId CheckpointStorage; TString GraphId; ICallbacks* ComputeActor = nullptr; diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index e81b183458..deaadefb42 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -1,7 +1,7 @@ #pragma once -#include "dq_compute_actor.h" -#include "dq_compute_actor_channels.h" +#include "dq_compute_actor.h" +#include "dq_compute_actor_channels.h" #include "dq_compute_actor_checkpoints.h" #include "dq_compute_actor_sinks.h" #include "dq_compute_actor_sources.h" @@ -82,24 +82,24 @@ public: if (RuntimeSettings.Timeout) { CA_LOG_D("Set execution timeout " << *RuntimeSettings.Timeout); - this->Schedule(*RuntimeSettings.Timeout, new NActors::TEvents::TEvWakeup(EEvWakeupTag::TimeoutTag)); + this->Schedule(*RuntimeSettings.Timeout, new NActors::TEvents::TEvWakeup(EEvWakeupTag::TimeoutTag)); } if (auto reportStatsSettings = RuntimeSettings.ReportStatsSettings) { if (reportStatsSettings->MaxInterval) { CA_LOG_D("Set periodic stats " << reportStatsSettings->MaxInterval); - this->Schedule(reportStatsSettings->MaxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); + this->Schedule(reportStatsSettings->MaxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); } } - if (SayHelloOnBootstrap()) { + if (SayHelloOnBootstrap()) { // say "Hello" to executer auto ev = MakeHolder<TEvDqCompute::TEvState>(); ev->Record.SetState(NDqProto::COMPUTE_STATE_EXECUTING); ev->Record.SetTaskId(Task.GetId()); this->Send(ExecuterId, ev.Release(), NActors::IEventHandle::FlagTrackDelivery); - this->Become(&TDqComputeActorBase::StateFuncBase); + this->Become(&TDqComputeActorBase::StateFuncBase); } static_cast<TDerived*>(this)->DoBootstrap(); @@ -188,8 +188,8 @@ protected: hFunc(TEvDqCompute::TEvResumeExecution, HandleExecuteBase); hFunc(TEvDqCompute::TEvChannelsInfo, HandleExecuteBase); hFunc(TEvDq::TEvAbortExecution, HandleExecuteBase); - hFunc(NActors::TEvents::TEvWakeup, HandleExecuteBase); - hFunc(NActors::TEvents::TEvUndelivered, HandleExecuteBase); + hFunc(NActors::TEvents::TEvWakeup, HandleExecuteBase); + hFunc(NActors::TEvents::TEvUndelivered, HandleExecuteBase); FFunc(TEvDqCompute::TEvChannelData::EventType, Channels->Receive); FFunc(TEvDqCompute::TEvChannelDataAck::EventType, Channels->Receive); hFunc(TEvDqCompute::TEvRun, HandleExecuteBase); @@ -221,7 +221,7 @@ protected: protected: void DoExecute() { - auto guard = BindAllocator(); + auto guard = BindAllocator(); auto* alloc = guard.GetMutex(); if (State == NDqProto::COMPUTE_STATE_FINISHED) { @@ -254,7 +254,7 @@ protected: ReportStats(now); } - virtual void DoExecuteImpl() { + virtual void DoExecuteImpl() { auto sourcesState = GetSourcesState(); PollSourceActors(); @@ -285,21 +285,21 @@ protected: } void ProcessOutputsImpl(ERunStatus status) { - ProcessOutputsState.LastRunStatus = status; + ProcessOutputsState.LastRunStatus = status; + + if (ProcessOutputsState.Inflight == 0) { + ProcessOutputsState = TProcessOutputsState(); + } - if (ProcessOutputsState.Inflight == 0) { - ProcessOutputsState = TProcessOutputsState(); - } - for (auto& entry : OutputChannelsMap) { const ui64 channelId = entry.first; TOutputChannelInfo& outputChannel = entry.second; if (!outputChannel.HasPeer) { // Channel info not complete, skip until dst info is available - ProcessOutputsState.ChannelsReady = false; - ProcessOutputsState.HasDataToSend = true; - ProcessOutputsState.AllOutputsFinished = false; + ProcessOutputsState.ChannelsReady = false; + ProcessOutputsState.HasDataToSend = true; + ProcessOutputsState.AllOutputsFinished = false; CA_LOG_D("Can not drain channelId: " << channelId << ", no dst actor id"); if (Y_UNLIKELY(outputChannel.Stats)) { outputChannel.Stats->NoDstActorId++; @@ -310,33 +310,33 @@ protected: if (!outputChannel.Finished || Checkpoints) { if (Channels->CanSendChannelData(channelId)) { auto peerState = Channels->GetOutputChannelInFlightState(channelId); - DrainOutputChannel(outputChannel, peerState); - } else { - ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; + DrainOutputChannel(outputChannel, peerState); + } else { + ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; } } else { CA_LOG_D("Do not drain channelId: " << channelId << ", finished"); - ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; + ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; } } for (auto& entry : SinksMap) { const ui64 outputIndex = entry.first; TSinkInfo& sinkInfo = entry.second; - DrainSink(outputIndex, sinkInfo); - } - - CheckRunStatus(); - } - - void CheckRunStatus() { - if (ProcessOutputsState.Inflight != 0) { - return; - } - - auto status = ProcessOutputsState.LastRunStatus; - - if (status == ERunStatus::PendingInput && ProcessOutputsState.AllOutputsFinished) { + DrainSink(outputIndex, sinkInfo); + } + + CheckRunStatus(); + } + + void CheckRunStatus() { + if (ProcessOutputsState.Inflight != 0) { + return; + } + + auto status = ProcessOutputsState.LastRunStatus; + + if (status == ERunStatus::PendingInput && ProcessOutputsState.AllOutputsFinished) { CA_LOG_D("All outputs have been finished. Consider finished"); status = ERunStatus::Finished; } @@ -349,10 +349,10 @@ protected: // and sends us a new batch of data. bool pollSent = false; for (auto& [channelId, inputChannel] : InputChannelsMap) { - pollSent |= Channels->PollChannel(channelId, GetInputChannelFreeSpace(channelId)); + pollSent |= Channels->PollChannel(channelId, GetInputChannelFreeSpace(channelId)); } if (!pollSent) { - if (ProcessOutputsState.DataWasSent) { + if (ProcessOutputsState.DataWasSent) { ContinueExecute(); } return; @@ -360,7 +360,7 @@ protected: } if (status == ERunStatus::PendingOutput) { - if (ProcessOutputsState.DataWasSent) { + if (ProcessOutputsState.DataWasSent) { // we have sent some data, so we have space in output channel(s) ContinueExecute(); } @@ -369,9 +369,9 @@ protected: // Handle finishing of our task. if (status == ERunStatus::Finished && State != NDqProto::COMPUTE_STATE_FINISHED) { - if (ProcessOutputsState.HasDataToSend || !ProcessOutputsState.ChannelsReady) { + if (ProcessOutputsState.HasDataToSend || !ProcessOutputsState.ChannelsReady) { CA_LOG_D("Continue execution, either output buffers are not empty or not all channels are ready" - << ", hasDataToSend: " << ProcessOutputsState.HasDataToSend << ", channelsReady: " << ProcessOutputsState.ChannelsReady); + << ", hasDataToSend: " << ProcessOutputsState.HasDataToSend << ", channelsReady: " << ProcessOutputsState.ChannelsReady); } else { if (!Channels->FinishInputChannels()) { CA_LOG_D("Continue execution, not all input channels are initialized"); @@ -396,13 +396,13 @@ protected: if (Channels) { TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Channels->SelfId(), this->SelfId(), - new NActors::TEvents::TEvPoison); + new NActors::TEvents::TEvPoison); Channels->Receive(handle, NActors::TActivationContext::AsActorContext()); } if (Checkpoints) { TAutoPtr<NActors::IEventHandle> handle = new NActors::IEventHandle(Checkpoints->SelfId(), this->SelfId(), - new NActors::TEvents::TEvPoison); + new NActors::TEvents::TEvPoison); Checkpoints->Receive(handle, NActors::TActivationContext::AsActorContext()); } @@ -419,9 +419,9 @@ protected: } for (auto& [_, outputChannel] : OutputChannelsMap) { - if (outputChannel.Channel) { - outputChannel.Channel->Terminate(); - } + if (outputChannel.Channel) { + outputChannel.Channel->Terminate(); + } } if (RuntimeSettings.TerminateHandler) { @@ -491,7 +491,7 @@ protected: CA_LOG_E(TIssuesIds::EIssueCode_Name(issueCode) << ": " << message << "."); TIssue issue(message); SetIssueCode(issueCode, issue); - std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard = MaybeBindAllocator(); + std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard = MaybeBindAllocator(); State = NDqProto::COMPUTE_STATE_FAILURE; ReportStateAndMaybeDie(std::move(issue)); } @@ -511,14 +511,14 @@ public: return inputChannel->Channel->GetFreeSpace(); } - void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) override { + void TakeInputChannelData(NDqProto::TChannelData&& channelData, bool ack) override { TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelData.GetChannelId()); YQL_ENSURE(inputChannel, "task: " << Task.GetId() << ", unknown input channelId: " << channelData.GetChannelId()); auto channel = inputChannel->Channel; if (channelData.GetData().GetRows()) { - auto guard = BindAllocator(); + auto guard = BindAllocator(); channel->Push(std::move(*channelData.MutableData())); } @@ -534,11 +534,11 @@ public: channel->Finish(); } - if (ack) { - Channels->SendChannelDataAck(channel->GetChannelId(), channel->GetFreeSpace()); - } - - ResumeExecution(); + if (ack) { + Channels->SendChannelDataAck(channel->GetChannelId(), channel->GetFreeSpace()); + } + + ResumeExecution(); } void PeerFinished(ui64 channelId) override { @@ -552,7 +552,7 @@ public: << " about to clear buffer"); { - auto guard = BindAllocator(); + auto guard = BindAllocator(); ui32 dropRows = outputChannel->Channel->Drop(); CA_LOG_I("task: " << Task.GetId() << ", output channelId: " << channelId << " finished prematurely, " @@ -571,7 +571,7 @@ public: Checkpoints->OnSinkStateSaved(std::move(state), outputIndex, checkpoint); } -protected: +protected: bool ReadyToCheckpoint() const override { for (auto& [id, channelInfo] : InputChannelsMap) { if (channelInfo.CheckpointingMode == NDqProto::CHECKPOINTING_MODE_DISABLED) { @@ -630,7 +630,7 @@ protected: void LoadState(const NDqProto::TComputeActorState& state) override { CA_LOG_D("Load state"); - auto guard = BindAllocator(); + auto guard = BindAllocator(); const NDqProto::TMiniKqlProgramState& mkqlProgramState = state.GetMiniKqlProgram(); const ui64 version = mkqlProgramState.GetData().GetStateData().GetVersion(); YQL_ENSURE(version && version <= ComputeActorCurrentStateVersion && version != ComputeActorNonProtobufStateVersion, "Unsupported state version: " << version); @@ -668,16 +668,16 @@ protected: protected: struct TInputChannelInfo { - ui64 ChannelId; + ui64 ChannelId; IDqInputChannel::TPtr Channel; bool HasPeer = false; std::optional<NDqProto::TCheckpoint> PendingCheckpoint; const NDqProto::ECheckpointingMode CheckpointingMode; - ui64 FreeSpace = 0; + ui64 FreeSpace = 0; - explicit TInputChannelInfo(ui64 channelId, NDqProto::ECheckpointingMode checkpointingMode) - : ChannelId(channelId) - , CheckpointingMode(checkpointingMode) + explicit TInputChannelInfo(ui64 channelId, NDqProto::ECheckpointingMode checkpointingMode) + : ChannelId(channelId) + , CheckpointingMode(checkpointingMode) { } @@ -699,29 +699,29 @@ protected: }; struct TSourceInfo { - ui64 Index; + ui64 Index; IDqSource::TPtr Source; IDqSourceActor* SourceActor = nullptr; NActors::IActor* Actor = nullptr; TIssuesBuffer IssuesBuffer; bool Finished = false; - i64 FreeSpace = 1; - bool PushStarted = false; + i64 FreeSpace = 1; + bool PushStarted = false; - TSourceInfo(ui64 index) : Index(index), IssuesBuffer(IssuesBufferSize) {} + TSourceInfo(ui64 index) : Index(index), IssuesBuffer(IssuesBufferSize) {} }; struct TOutputChannelInfo { - ui64 ChannelId; + ui64 ChannelId; IDqOutputChannel::TPtr Channel; bool HasPeer = false; bool Finished = false; // != Channel->IsFinished() // If channel is in finished state, it sends only checkpoints. - bool PopStarted = false; + bool PopStarted = false; + + explicit TOutputChannelInfo(ui64 channelId) + : ChannelId(channelId) + { } - explicit TOutputChannelInfo(ui64 channelId) - : ChannelId(channelId) - { } - struct TStats { ui64 BlockedByCapacity = 0; ui64 NoDstActorId = 0; @@ -735,8 +735,8 @@ protected: NActors::IActor* Actor = nullptr; bool Finished = false; // If sink is in finished state, it receives only checkpoints. TIssuesBuffer IssuesBuffer; - bool PopStarted = false; - i64 SinkActorFreeSpaceBeforeSend = 0; + bool PopStarted = false; + i64 SinkActorFreeSpaceBeforeSend = 0; TSinkInfo() : IssuesBuffer(IssuesBufferSize) {} }; @@ -754,34 +754,34 @@ protected: virtual void TerminateSources(const TString& /* message */, bool /* success */) { } - virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator() { - return TaskRunner->BindAllocator(); - } - - virtual std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> MaybeBindAllocator() { - std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard; - if (!TaskRunner->GetTypeEnv().GetAllocator().IsAttached()) { - guard.emplace(TaskRunner->BindAllocator()); - } - return guard; - } - - virtual void SourcePush(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, TSourceInfo& source, i64 space, bool finished) { - source.Source->Push(std::move(batch), space); - if (finished) { - source.Source->Finish(); - source.Finished = true; - } - } - - virtual i64 SourceFreeSpace(TSourceInfo& source) { - return source.Source->GetFreeSpace(); - } - - virtual bool SayHelloOnBootstrap() { - return true; - } - + virtual TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator() { + return TaskRunner->BindAllocator(); + } + + virtual std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> MaybeBindAllocator() { + std::optional<TGuard<NKikimr::NMiniKQL::TScopedAlloc>> guard; + if (!TaskRunner->GetTypeEnv().GetAllocator().IsAttached()) { + guard.emplace(TaskRunner->BindAllocator()); + } + return guard; + } + + virtual void SourcePush(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, TSourceInfo& source, i64 space, bool finished) { + source.Source->Push(std::move(batch), space); + if (finished) { + source.Source->Finish(); + source.Finished = true; + } + } + + virtual i64 SourceFreeSpace(TSourceInfo& source) { + return source.Source->GetFreeSpace(); + } + + virtual bool SayHelloOnBootstrap() { + return true; + } + protected: void HandleExecuteBase(TEvDqCompute::TEvResumeExecution::TPtr&) { ResumeEventScheduled = false; @@ -798,7 +798,7 @@ protected: for (auto& channelUpdate : record.GetUpdate()) { TInputChannelInfo* inputChannel = InputChannelsMap.FindPtr(channelUpdate.GetId()); if (inputChannel && !inputChannel->HasPeer && channelUpdate.GetSrcEndpoint().HasActorId()) { - auto peer = NActors::ActorIdFromProto(channelUpdate.GetSrcEndpoint().GetActorId()); + auto peer = NActors::ActorIdFromProto(channelUpdate.GetSrcEndpoint().GetActorId()); CA_LOG_D("Update input channelId: " << channelUpdate.GetId() << ", peer: " << peer); @@ -810,7 +810,7 @@ protected: TOutputChannelInfo* outputChannel = OutputChannelsMap.FindPtr(channelUpdate.GetId()); if (outputChannel && !outputChannel->HasPeer && channelUpdate.GetDstEndpoint().HasActorId()) { - auto peer = NActors::ActorIdFromProto(channelUpdate.GetDstEndpoint().GetActorId()); + auto peer = NActors::ActorIdFromProto(channelUpdate.GetDstEndpoint().GetActorId()); CA_LOG_D("Update output channelId: " << channelUpdate.GetId() << ", peer: " << peer); @@ -826,7 +826,7 @@ protected: DoExecute(); } - void HandleExecuteBase(NActors::TEvents::TEvWakeup::TPtr& ev) { + void HandleExecuteBase(NActors::TEvents::TEvWakeup::TPtr& ev) { auto tag = (EEvWakeupTag) ev->Get()->Tag; switch (tag) { case EEvWakeupTag::TimeoutTag: { @@ -842,7 +842,7 @@ protected: } case EEvWakeupTag::PeriodicStatsTag: { const auto maxInterval = RuntimeSettings.ReportStatsSettings->MaxInterval; - this->Schedule(maxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); + this->Schedule(maxInterval, new NActors::TEvents::TEvWakeup(EEvWakeupTag::PeriodicStatsTag)); auto now = NActors::TActivationContext::Now(); if (now - LastSendStatsTime >= maxInterval) { @@ -859,7 +859,7 @@ protected: CA_LOG_E("Unhandled wakeup tag " << (ui64)tag); } - void HandleExecuteBase(NActors::TEvents::TEvUndelivered::TPtr& ev) { + void HandleExecuteBase(NActors::TEvents::TEvUndelivered::TPtr& ev) { ui32 lostEventType = ev->Get()->SourceType; switch (lostEventType) { case TEvDqCompute::TEvState::EventType: { @@ -942,9 +942,9 @@ protected: return allowedOvercommit; } -private: - - virtual void DrainOutputChannel(TOutputChannelInfo& outputChannel, const TDqComputeActorChannels::TPeerState& peerState) { +private: + + virtual void DrainOutputChannel(TOutputChannelInfo& outputChannel, const TDqComputeActorChannels::TPeerState& peerState) { YQL_ENSURE(!outputChannel.Finished || Checkpoints); const bool wasFinished = outputChannel.Finished; @@ -964,9 +964,9 @@ private: << ", toSend: " << toSend << ", finished: " << outputChannel.Channel->IsFinished()); - ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; - ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; - + ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; + ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; + if (toSend <= 0) { if (Y_UNLIKELY(outputChannel.Stats)) { outputChannel.Stats->BlockedByCapacity++; @@ -982,9 +982,9 @@ private: remains -= sent; } - ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; - ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; - ProcessOutputsState.DataWasSent |= (!wasFinished && outputChannel.Finished) || remains != toSend; + ProcessOutputsState.HasDataToSend |= !outputChannel.Finished; + ProcessOutputsState.AllOutputsFinished &= outputChannel.Finished; + ProcessOutputsState.DataWasSent |= (!wasFinished && outputChannel.Finished) || remains != toSend; } ui32 SendChannelDataChunk(TOutputChannelInfo& outputChannel, ui64 bytes) { @@ -1027,10 +1027,10 @@ private: return 0; } - virtual void DrainSink(ui64 outputIndex, TSinkInfo& sinkInfo) { - ProcessOutputsState.AllOutputsFinished &= sinkInfo.Finished; + virtual void DrainSink(ui64 outputIndex, TSinkInfo& sinkInfo) { + ProcessOutputsState.AllOutputsFinished &= sinkInfo.Finished; if (sinkInfo.Finished && !Checkpoints) { - return; + return; } Y_VERIFY(sinkInfo.Sink); @@ -1061,8 +1061,8 @@ private: << ". Free space decreased: " << (sinkActorFreeSpaceBeforeSend - sinkInfo.SinkActor->GetFreeSpace()) << ", sent data from buffer: " << sent); - ProcessOutputsState.HasDataToSend |= !sinkInfo.Finished; - ProcessOutputsState.DataWasSent |= sinkInfo.Finished || sent; + ProcessOutputsState.HasDataToSend |= !sinkInfo.Finished; + ProcessOutputsState.DataWasSent |= sinkInfo.Finished || sent; } ui32 SendSinkDataChunk(ui64 outputIndex, TSinkInfo& sinkInfo, ui64 bytes) { @@ -1103,7 +1103,7 @@ protected: return RuntimeSettings.RlPath; } - TTxId GetTxId() const { + TTxId GetTxId() const { return TxId; } @@ -1146,26 +1146,26 @@ protected: TaskRunner->Prepare(Task, limits, execCtx); - FillChannelMaps( - TaskRunner->GetHolderFactory(), - TaskRunner->GetTypeEnv(), - TaskRunner->GetSecureParams(), - TaskRunner->GetTaskParams()); - } - - void FillChannelMaps( - const NKikimr::NMiniKQL::THolderFactory& holderFactory, - const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, - const THashMap<TString, TString>& secureParams, - const THashMap<TString, TString>& taskParams) - { - if (TaskRunner) { - for (auto& [channelId, channel] : InputChannelsMap) { - channel.Channel = TaskRunner->GetInputChannel(channelId); - } + FillChannelMaps( + TaskRunner->GetHolderFactory(), + TaskRunner->GetTypeEnv(), + TaskRunner->GetSecureParams(), + TaskRunner->GetTaskParams()); + } + + void FillChannelMaps( + const NKikimr::NMiniKQL::THolderFactory& holderFactory, + const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, + const THashMap<TString, TString>& secureParams, + const THashMap<TString, TString>& taskParams) + { + if (TaskRunner) { + for (auto& [channelId, channel] : InputChannelsMap) { + channel.Channel = TaskRunner->GetInputChannel(channelId); + } } for (auto& [inputIndex, source] : SourcesMap) { - if (TaskRunner) { source.Source = TaskRunner->GetSource(inputIndex); Y_VERIFY(source.Source);} + if (TaskRunner) { source.Source = TaskRunner->GetSource(inputIndex); Y_VERIFY(source.Source);} Y_VERIFY(SourceActorFactory); const auto& inputDesc = Task.GetInputs(inputIndex); const ui64 i = inputIndex; // Crutch for clang @@ -1175,21 +1175,21 @@ protected: .InputDesc = inputDesc, .InputIndex = inputIndex, .TxId = TxId, - .SecureParams = secureParams, - .TaskParams = taskParams, + .SecureParams = secureParams, + .TaskParams = taskParams, .Callback = this, - .TypeEnv = typeEnv, - .HolderFactory = holderFactory + .TypeEnv = typeEnv, + .HolderFactory = holderFactory }); this->RegisterWithSameMailbox(source.Actor); } - if (TaskRunner) { - for (auto& [channelId, channel] : OutputChannelsMap) { - channel.Channel = TaskRunner->GetOutputChannel(channelId); - } + if (TaskRunner) { + for (auto& [channelId, channel] : OutputChannelsMap) { + channel.Channel = TaskRunner->GetOutputChannel(channelId); + } } for (auto& [outputIndex, sink] : SinksMap) { - if (TaskRunner) { sink.Sink = TaskRunner->GetSink(outputIndex); } + if (TaskRunner) { sink.Sink = TaskRunner->GetSink(outputIndex); } Y_VERIFY(SinkActorFactory); const auto& outputDesc = Task.GetOutputs(outputIndex); const ui64 i = outputIndex; // Crutch for clang @@ -1199,10 +1199,10 @@ protected: .OutputDesc = outputDesc, .OutputIndex = outputIndex, .TxId = TxId, - .SecureParams = secureParams, + .SecureParams = secureParams, .Callback = this, - .TypeEnv = typeEnv, - .HolderFactory = holderFactory + .TypeEnv = typeEnv, + .HolderFactory = holderFactory }); this->RegisterWithSameMailbox(sink.Actor); } @@ -1215,13 +1215,13 @@ protected: } for (auto& [inputIndex, source] : SourcesMap) { - Y_VERIFY(!TaskRunner || source.Source); + Y_VERIFY(!TaskRunner || source.Source); if (source.Finished) { const ui64 indexForLogging = inputIndex; // Crutch for clang CA_LOG_D("Skip polling source[" << indexForLogging << "]: finished"); continue; } - const i64 freeSpace = SourceFreeSpace(source); + const i64 freeSpace = SourceFreeSpace(source); if (freeSpace > 0) { NKikimr::NMiniKQL::TUnboxedValueVector batch; Y_VERIFY(source.SourceActor); @@ -1232,7 +1232,7 @@ protected: << ". Buffer free space: " << freeSpace << ", read from source: " << space << " bytes, " << batch.size() << " rows, finished: " << finished); - SourcePush(std::move(batch), source, space, finished); + SourcePush(std::move(batch), source, space, finished); } } } @@ -1277,11 +1277,11 @@ private: const auto& inputDesc = Task.GetInputs(i); Y_VERIFY(!inputDesc.HasSource() || inputDesc.ChannelsSize() == 0); // HasSource => no channels if (inputDesc.HasSource()) { - auto result = SourcesMap.emplace(i, TSourceInfo(i)); + auto result = SourcesMap.emplace(i, TSourceInfo(i)); YQL_ENSURE(result.second); } else { for (auto& channel : inputDesc.GetChannels()) { - auto result = InputChannelsMap.emplace(channel.GetId(), TInputChannelInfo(channel.GetId(), channel.GetCheckpointingMode())); + auto result = InputChannelsMap.emplace(channel.GetId(), TInputChannelInfo(channel.GetId(), channel.GetCheckpointingMode())); YQL_ENSURE(result.second); } } @@ -1296,7 +1296,7 @@ private: YQL_ENSURE(result.second); } else { for (auto& channel : outputDesc.GetChannels()) { - TOutputChannelInfo outputChannel(channel.GetId()); + TOutputChannelInfo outputChannel(channel.GetId()); outputChannel.HasPeer = channel.GetDstEndpoint().HasActorId(); if (Y_UNLIKELY(RuntimeSettings.StatsMode >= NDqProto::DQ_STATS_MODE_PROFILE)) { @@ -1403,11 +1403,11 @@ private: protoOutputChannelStats.SetResentMessages(x->ResentMessages); } - if (auto* outputInfo = OutputChannelsMap.FindPtr(protoOutputChannelStats.GetChannelId())) { - if (auto *x = outputInfo->Stats.Get()) { - protoOutputChannelStats.SetBlockedByCapacity(x->BlockedByCapacity); - protoOutputChannelStats.SetNoDstActorId(x->NoDstActorId); - } + if (auto* outputInfo = OutputChannelsMap.FindPtr(protoOutputChannelStats.GetChannelId())) { + if (auto *x = outputInfo->Stats.Get()) { + protoOutputChannelStats.SetBlockedByCapacity(x->BlockedByCapacity); + protoOutputChannelStats.SetNoDstActorId(x->NoDstActorId); + } } } } @@ -1457,7 +1457,7 @@ private: protected: const NActors::TActorId ExecuterId; - const TTxId TxId; + const TTxId TxId; const NDqProto::TDqTask Task; const TComputeRuntimeSettings RuntimeSettings; const TComputeMemoryLimits MemoryLimits; @@ -1487,16 +1487,16 @@ protected: std::unique_ptr<TBasicStats> BasicStats; std::unique_ptr<TProfileStats> ProfileStats; - struct TProcessOutputsState { - int Inflight = 0; - bool ChannelsReady = true; - bool HasDataToSend = false; - bool DataWasSent = false; - bool AllOutputsFinished = true; - ERunStatus LastRunStatus = ERunStatus::PendingInput; - }; - TProcessOutputsState ProcessOutputsState; - + struct TProcessOutputsState { + int Inflight = 0; + bool ChannelsReady = true; + bool HasDataToSend = false; + bool DataWasSent = false; + bool AllOutputsFinished = true; + ERunStatus LastRunStatus = ERunStatus::PendingInput; + }; + TProcessOutputsState ProcessOutputsState; + private: bool Running = true; TInstant LastSendStatsTime; diff --git a/ydb/library/yql/dq/actors/compute/ya.make b/ydb/library/yql/dq/actors/compute/ya.make index 217fb7fe2b..c4b20bd502 100644 --- a/ydb/library/yql/dq/actors/compute/ya.make +++ b/ydb/library/yql/dq/actors/compute/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - spuchin - g:kikimr -) - -SRCS( - dq_compute_actor.cpp - dq_async_compute_actor.cpp - dq_compute_actor_channels.cpp +LIBRARY() + +OWNER( + spuchin + g:kikimr +) + +SRCS( + dq_compute_actor.cpp + dq_async_compute_actor.cpp + dq_compute_actor_channels.cpp dq_compute_actor_checkpoints.cpp dq_compute_actor_io_actors_factory.cpp dq_compute_actor_stats.cpp dq_compute_issues_buffer.cpp retry_queue.cpp -) - -PEERDIR( - library/cpp/actors/core +) + +PEERDIR( + library/cpp/actors/core ydb/core/base ydb/core/kqp/common ydb/core/kqp/runtime @@ -30,12 +30,12 @@ PEERDIR( ydb/library/yql/dq/tasks ydb/library/yql/minikql/comp_nodes ydb/library/yql/minikql/computation -) - +) + YQL_LAST_ABI_VERSION() - + END() - + RECURSE_FOR_TESTS( ut ) diff --git a/ydb/library/yql/dq/actors/dq_events_ids.cpp b/ydb/library/yql/dq/actors/dq_events_ids.cpp index c1aa2efe68..8c82d46fd9 100644 --- a/ydb/library/yql/dq/actors/dq_events_ids.cpp +++ b/ydb/library/yql/dq/actors/dq_events_ids.cpp @@ -1 +1 @@ -#include "dq_events_ids.h" +#include "dq_events_ids.h" diff --git a/ydb/library/yql/dq/actors/dq_events_ids.h b/ydb/library/yql/dq/actors/dq_events_ids.h index 2d511a7c0b..698377a393 100644 --- a/ydb/library/yql/dq/actors/dq_events_ids.h +++ b/ydb/library/yql/dq/actors/dq_events_ids.h @@ -1,12 +1,12 @@ #pragma once -#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/events.h> namespace NYql { namespace NDq { -struct TDqEvents { - enum EEventSpaceDq { +struct TDqEvents { + enum EEventSpaceDq { ES_DQ_COMPUTE_KQP_COMPATIBLE = 4145, // TKikimrEvents::ES_KQP ES_DQ_COMPUTE = 4212 //TKikimrEvents::ES_DQ }; diff --git a/ydb/library/yql/dq/actors/protos/ya.make b/ydb/library/yql/dq/actors/protos/ya.make index 969aa1c44c..81e25d0336 100644 --- a/ydb/library/yql/dq/actors/protos/ya.make +++ b/ydb/library/yql/dq/actors/protos/ya.make @@ -1,21 +1,21 @@ -PROTO_LIBRARY() - -OWNER(g:yql) - -SRCS( - dq_events.proto +PROTO_LIBRARY() + +OWNER(g:yql) + +SRCS( + dq_events.proto dq_stats.proto -) - -PEERDIR( - library/cpp/actors/protos +) + +PEERDIR( + library/cpp/actors/protos ydb/public/api/protos ydb/library/yql/core/issue/protos ydb/library/yql/dq/proto ydb/library/yql/public/issue/protos ydb/library/yql/public/types -) - -EXCLUDE_TAGS(GO_PROTO) - -END() +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/library/yql/dq/actors/task_runner/events.cpp b/ydb/library/yql/dq/actors/task_runner/events.cpp index a4930159cf..6c3d2603e7 100644 --- a/ydb/library/yql/dq/actors/task_runner/events.cpp +++ b/ydb/library/yql/dq/actors/task_runner/events.cpp @@ -1 +1 @@ -#include "events.h" +#include "events.h" diff --git a/ydb/library/yql/dq/actors/task_runner/events.h b/ydb/library/yql/dq/actors/task_runner/events.h index e894e202c8..ae5894cddd 100644 --- a/ydb/library/yql/dq/actors/task_runner/events.h +++ b/ydb/library/yql/dq/actors/task_runner/events.h @@ -1,308 +1,308 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/event_pb.h> - +#pragma once + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/event_pb.h> + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node.h> - + #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> #include <ydb/library/yql/dq/common/dq_common.h> #include <ydb/library/yql/dq/proto/dq_transport.pb.h> #include <ydb/library/yql/dq/proto/dq_tasks.pb.h> - - -namespace NYql::NDq { - -namespace NTaskRunnerActor { - -struct TTaskRunnerEvents { - enum { - ES_CREATE = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 20000, - ES_CREATE_FINISHED, - // channel->Pop -> TEvChannelPopFinished - ES_POP, - ES_POP_FINISHED, - // channel->Push -> TEvContinueRun - ES_PUSH, - ES_CONTINUE_RUN, - // ES_CONTINUE_RUN -> TaskRunner->Run() -> TEvTaskRunFinished - ES_RUN_FINISHED, - - ES_SOURCE_PUSH, - ES_SOURCE_PUSH_FINISHED, - - ES_SINK_POP, - ES_SINK_POP_FINISHED, - - ES_ERROR - }; -}; - -struct TTaskRunnerActorSensorEntry { - TString Name; - i64 Sum = 0; - i64 Max = 0; - i64 Min = 0; - i64 Avg = 0; - i64 Count = 0; -}; - -struct TTaskRunnerActorRusage { - i64 Utime = 0; - i64 Stime = 0; - i64 MajorPageFaults = 0; -}; - -using TTaskRunnerActorSensors = TVector<TTaskRunnerActorSensorEntry>; - -struct TEvError - : NActors::TEventLocal<TEvError, TTaskRunnerEvents::ES_ERROR> -{ - struct TStatus { - int ExitCode; - TString Stderr; - }; - - TEvError() = default; - - TEvError(const TString& message, bool retriable = false, bool fallback = false) - : Message(message) - , Retriable(retriable) - , Fallback(fallback) - { } - - TEvError(const TString& message, TStatus status, bool retriable = false, bool fallback = false) - : Message(message) - , Retriable(retriable) - , Fallback(fallback) - , Status(status) - { } - - TString Message; - bool Retriable; - bool Fallback; - TMaybe<TStatus> Status = Nothing(); -}; - -struct TEvPop - : NActors::TEventLocal<TEvPop, TTaskRunnerEvents::ES_POP> -{ - TEvPop() = default; - TEvPop(ui32 channelId, bool wasFinished, i64 toPop) - : ChannelId(channelId) - , WasFinished(wasFinished) - , Size(toPop) - { } - TEvPop(ui32 channelId) - : ChannelId(channelId) - , WasFinished(false) - , Size(0) - { } - - const ui32 ChannelId; - const bool WasFinished; - const i64 Size; -}; - -struct TEvPush - : NActors::TEventLocal<TEvPush, TTaskRunnerEvents::ES_PUSH> -{ - TEvPush() = default; - TEvPush(ui32 channelId, bool finish = true, bool askFreeSpace = false) - : ChannelId(channelId) - , HasData(false) - , Finish(finish) - , AskFreeSpace(askFreeSpace) - { } - TEvPush(ui32 channelId, NDqProto::TData&& data, bool finish = false, bool askFreeSpace = false) - : ChannelId(channelId) - , HasData(true) - , Finish(finish) - , AskFreeSpace(askFreeSpace) - , Data(std::move(data)) - { } - - const ui32 ChannelId; - const bool HasData = false; - const bool Finish = false; - const bool AskFreeSpace = false; - NDqProto::TData Data; -}; - -struct TEvTaskRunnerCreate - : NActors::TEventLocal<TEvTaskRunnerCreate, TTaskRunnerEvents::ES_CREATE> -{ - TEvTaskRunnerCreate() = default; - TEvTaskRunnerCreate( - const NDqProto::TDqTask& task, - const TDqTaskRunnerMemoryLimits& memoryLimits, - const std::shared_ptr<IDqTaskRunnerExecutionContext>& execCtx = std::shared_ptr<IDqTaskRunnerExecutionContext>(new TDqTaskRunnerExecutionContext()), - const TDqTaskRunnerParameterProvider& parameterProvider = {}) - : Task(task) - , MemoryLimits(memoryLimits) - , ExecCtx(execCtx) - , ParameterProvider(parameterProvider) - { } - - NDqProto::TDqTask Task; - TDqTaskRunnerMemoryLimits MemoryLimits; - std::shared_ptr<IDqTaskRunnerExecutionContext> ExecCtx; - TDqTaskRunnerParameterProvider ParameterProvider; -}; - -struct TEvTaskRunnerCreateFinished - : NActors::TEventLocal<TEvTaskRunnerCreateFinished, TTaskRunnerEvents::ES_CREATE_FINISHED> -{ - - TEvTaskRunnerCreateFinished() = default; - TEvTaskRunnerCreateFinished( - const THashMap<TString, TString>& secureParams, - const THashMap<TString, TString>& taskParams, - const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, - const NKikimr::NMiniKQL::THolderFactory& holderFactory, - const TTaskRunnerActorSensors& sensors = {}) - : Sensors(sensors) - , SecureParams(secureParams) - , TaskParams(taskParams) - , TypeEnv(typeEnv) - , HolderFactory(holderFactory) - { } - - TTaskRunnerActorSensors Sensors; - - // for sources/sinks - const THashMap<TString, TString>& SecureParams; - const THashMap<TString, TString>& TaskParams; - const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; - const NKikimr::NMiniKQL::THolderFactory& HolderFactory; -}; - -struct TEvTaskRunFinished - : NActors::TEventLocal<TEvTaskRunFinished, TTaskRunnerEvents::ES_RUN_FINISHED> -{ - TEvTaskRunFinished() = default; - TEvTaskRunFinished( - NDq::ERunStatus runStatus, - THashMap<ui32, ui64>&& inputMap, - THashMap<ui32, ui64>&& sourcesMap, - const TTaskRunnerActorSensors& sensors = {}, - const TTaskRunnerActorRusage& rusage = {}) - : RunStatus(runStatus) - , Sensors(sensors) - , Rusage(rusage) - , InputChannelFreeSpace(std::move(inputMap)) - , SourcesFreeSpace(std::move(sourcesMap)) - { } - - NDq::ERunStatus RunStatus; - TTaskRunnerActorSensors Sensors; - TTaskRunnerActorRusage Rusage; - - THashMap<ui32, ui64> InputChannelFreeSpace; - THashMap<ui32, ui64> SourcesFreeSpace; -}; - -struct TEvChannelPopFinished - : NActors::TEventLocal<TEvChannelPopFinished, TTaskRunnerEvents::ES_POP_FINISHED> { - TEvChannelPopFinished() = default; - TEvChannelPopFinished(ui32 channelId) - : ChannelId(channelId) - { } - TEvChannelPopFinished(ui32 channelId, TVector<NDqProto::TData>&& data, bool finished, bool changed, const TTaskRunnerActorSensors& sensors = {}) - : Sensors(sensors) - , ChannelId(channelId) - , Data(std::move(data)) - , Finished(finished) - , Changed(changed) - { } - - TTaskRunnerActorSensors Sensors; - - const ui32 ChannelId; - TVector<NDqProto::TData> Data; - bool Finished; - bool Changed; -}; - -struct TEvContinueRun - : NActors::TEventLocal<TEvContinueRun, TTaskRunnerEvents::ES_CONTINUE_RUN> { - - TEvContinueRun() = default; - TEvContinueRun(ui32 channelId, ui64 freeSpace) - : ChannelId(channelId) - , AskFreeSpace(false) - , MemLimit(0) - , FreeSpace(freeSpace) - { } - TEvContinueRun(THashSet<ui32>&& inputChannels, ui64 memLimit) - : ChannelId(0) - , AskFreeSpace(false) - , InputChannels(std::move(inputChannels)) - , MemLimit(memLimit) - { } - - ui32 ChannelId; - bool AskFreeSpace = true; - const THashSet<ui32> InputChannels; - ui64 MemLimit; - ui64 FreeSpace; -}; - -struct TEvSourcePushFinished - : NActors::TEventLocal<TEvSourcePushFinished, TTaskRunnerEvents::ES_SOURCE_PUSH_FINISHED> -{ - TEvSourcePushFinished() = default; - TEvSourcePushFinished(ui64 index) - : Index(index) - { } - - ui64 Index; -}; - -struct TEvSinkPop - : NActors::TEventLocal<TEvSinkPop, TTaskRunnerEvents::ES_SINK_POP> -{ - TEvSinkPop() = default; - TEvSinkPop(ui64 index, i64 size) - : Index(index) - , Size(size) - { } - - ui64 Index; - i64 Size; -}; - -struct TEvSinkPopFinished - : NActors::TEventLocal<TEvSinkPopFinished, TTaskRunnerEvents::ES_SINK_POP_FINISHED> -{ - TEvSinkPopFinished() = default; - TEvSinkPopFinished( - ui64 index, - TMaybe<NDqProto::TCheckpoint>&& checkpoint, - i64 size, - i64 checkpointSize, - bool finished, - bool changed) - : Index(index) - , Checkpoint(std::move(checkpoint)) - , Size(size) - , CheckpointSize(checkpointSize) - , Finished(finished) - , Changed(changed) - { } - - const ui64 Index; - TVector<TString> Strings; - TMaybe<NDqProto::TCheckpoint> Checkpoint; - i64 Size; - i64 CheckpointSize; - bool Finished; - bool Changed; -}; - -} // namespace NTaskRunnerActor - -} // namespace NYql::NDq + + +namespace NYql::NDq { + +namespace NTaskRunnerActor { + +struct TTaskRunnerEvents { + enum { + ES_CREATE = EventSpaceBegin(NActors::TEvents::EEventSpace::ES_USERSPACE) + 20000, + ES_CREATE_FINISHED, + // channel->Pop -> TEvChannelPopFinished + ES_POP, + ES_POP_FINISHED, + // channel->Push -> TEvContinueRun + ES_PUSH, + ES_CONTINUE_RUN, + // ES_CONTINUE_RUN -> TaskRunner->Run() -> TEvTaskRunFinished + ES_RUN_FINISHED, + + ES_SOURCE_PUSH, + ES_SOURCE_PUSH_FINISHED, + + ES_SINK_POP, + ES_SINK_POP_FINISHED, + + ES_ERROR + }; +}; + +struct TTaskRunnerActorSensorEntry { + TString Name; + i64 Sum = 0; + i64 Max = 0; + i64 Min = 0; + i64 Avg = 0; + i64 Count = 0; +}; + +struct TTaskRunnerActorRusage { + i64 Utime = 0; + i64 Stime = 0; + i64 MajorPageFaults = 0; +}; + +using TTaskRunnerActorSensors = TVector<TTaskRunnerActorSensorEntry>; + +struct TEvError + : NActors::TEventLocal<TEvError, TTaskRunnerEvents::ES_ERROR> +{ + struct TStatus { + int ExitCode; + TString Stderr; + }; + + TEvError() = default; + + TEvError(const TString& message, bool retriable = false, bool fallback = false) + : Message(message) + , Retriable(retriable) + , Fallback(fallback) + { } + + TEvError(const TString& message, TStatus status, bool retriable = false, bool fallback = false) + : Message(message) + , Retriable(retriable) + , Fallback(fallback) + , Status(status) + { } + + TString Message; + bool Retriable; + bool Fallback; + TMaybe<TStatus> Status = Nothing(); +}; + +struct TEvPop + : NActors::TEventLocal<TEvPop, TTaskRunnerEvents::ES_POP> +{ + TEvPop() = default; + TEvPop(ui32 channelId, bool wasFinished, i64 toPop) + : ChannelId(channelId) + , WasFinished(wasFinished) + , Size(toPop) + { } + TEvPop(ui32 channelId) + : ChannelId(channelId) + , WasFinished(false) + , Size(0) + { } + + const ui32 ChannelId; + const bool WasFinished; + const i64 Size; +}; + +struct TEvPush + : NActors::TEventLocal<TEvPush, TTaskRunnerEvents::ES_PUSH> +{ + TEvPush() = default; + TEvPush(ui32 channelId, bool finish = true, bool askFreeSpace = false) + : ChannelId(channelId) + , HasData(false) + , Finish(finish) + , AskFreeSpace(askFreeSpace) + { } + TEvPush(ui32 channelId, NDqProto::TData&& data, bool finish = false, bool askFreeSpace = false) + : ChannelId(channelId) + , HasData(true) + , Finish(finish) + , AskFreeSpace(askFreeSpace) + , Data(std::move(data)) + { } + + const ui32 ChannelId; + const bool HasData = false; + const bool Finish = false; + const bool AskFreeSpace = false; + NDqProto::TData Data; +}; + +struct TEvTaskRunnerCreate + : NActors::TEventLocal<TEvTaskRunnerCreate, TTaskRunnerEvents::ES_CREATE> +{ + TEvTaskRunnerCreate() = default; + TEvTaskRunnerCreate( + const NDqProto::TDqTask& task, + const TDqTaskRunnerMemoryLimits& memoryLimits, + const std::shared_ptr<IDqTaskRunnerExecutionContext>& execCtx = std::shared_ptr<IDqTaskRunnerExecutionContext>(new TDqTaskRunnerExecutionContext()), + const TDqTaskRunnerParameterProvider& parameterProvider = {}) + : Task(task) + , MemoryLimits(memoryLimits) + , ExecCtx(execCtx) + , ParameterProvider(parameterProvider) + { } + + NDqProto::TDqTask Task; + TDqTaskRunnerMemoryLimits MemoryLimits; + std::shared_ptr<IDqTaskRunnerExecutionContext> ExecCtx; + TDqTaskRunnerParameterProvider ParameterProvider; +}; + +struct TEvTaskRunnerCreateFinished + : NActors::TEventLocal<TEvTaskRunnerCreateFinished, TTaskRunnerEvents::ES_CREATE_FINISHED> +{ + + TEvTaskRunnerCreateFinished() = default; + TEvTaskRunnerCreateFinished( + const THashMap<TString, TString>& secureParams, + const THashMap<TString, TString>& taskParams, + const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, + const NKikimr::NMiniKQL::THolderFactory& holderFactory, + const TTaskRunnerActorSensors& sensors = {}) + : Sensors(sensors) + , SecureParams(secureParams) + , TaskParams(taskParams) + , TypeEnv(typeEnv) + , HolderFactory(holderFactory) + { } + + TTaskRunnerActorSensors Sensors; + + // for sources/sinks + const THashMap<TString, TString>& SecureParams; + const THashMap<TString, TString>& TaskParams; + const NKikimr::NMiniKQL::TTypeEnvironment& TypeEnv; + const NKikimr::NMiniKQL::THolderFactory& HolderFactory; +}; + +struct TEvTaskRunFinished + : NActors::TEventLocal<TEvTaskRunFinished, TTaskRunnerEvents::ES_RUN_FINISHED> +{ + TEvTaskRunFinished() = default; + TEvTaskRunFinished( + NDq::ERunStatus runStatus, + THashMap<ui32, ui64>&& inputMap, + THashMap<ui32, ui64>&& sourcesMap, + const TTaskRunnerActorSensors& sensors = {}, + const TTaskRunnerActorRusage& rusage = {}) + : RunStatus(runStatus) + , Sensors(sensors) + , Rusage(rusage) + , InputChannelFreeSpace(std::move(inputMap)) + , SourcesFreeSpace(std::move(sourcesMap)) + { } + + NDq::ERunStatus RunStatus; + TTaskRunnerActorSensors Sensors; + TTaskRunnerActorRusage Rusage; + + THashMap<ui32, ui64> InputChannelFreeSpace; + THashMap<ui32, ui64> SourcesFreeSpace; +}; + +struct TEvChannelPopFinished + : NActors::TEventLocal<TEvChannelPopFinished, TTaskRunnerEvents::ES_POP_FINISHED> { + TEvChannelPopFinished() = default; + TEvChannelPopFinished(ui32 channelId) + : ChannelId(channelId) + { } + TEvChannelPopFinished(ui32 channelId, TVector<NDqProto::TData>&& data, bool finished, bool changed, const TTaskRunnerActorSensors& sensors = {}) + : Sensors(sensors) + , ChannelId(channelId) + , Data(std::move(data)) + , Finished(finished) + , Changed(changed) + { } + + TTaskRunnerActorSensors Sensors; + + const ui32 ChannelId; + TVector<NDqProto::TData> Data; + bool Finished; + bool Changed; +}; + +struct TEvContinueRun + : NActors::TEventLocal<TEvContinueRun, TTaskRunnerEvents::ES_CONTINUE_RUN> { + + TEvContinueRun() = default; + TEvContinueRun(ui32 channelId, ui64 freeSpace) + : ChannelId(channelId) + , AskFreeSpace(false) + , MemLimit(0) + , FreeSpace(freeSpace) + { } + TEvContinueRun(THashSet<ui32>&& inputChannels, ui64 memLimit) + : ChannelId(0) + , AskFreeSpace(false) + , InputChannels(std::move(inputChannels)) + , MemLimit(memLimit) + { } + + ui32 ChannelId; + bool AskFreeSpace = true; + const THashSet<ui32> InputChannels; + ui64 MemLimit; + ui64 FreeSpace; +}; + +struct TEvSourcePushFinished + : NActors::TEventLocal<TEvSourcePushFinished, TTaskRunnerEvents::ES_SOURCE_PUSH_FINISHED> +{ + TEvSourcePushFinished() = default; + TEvSourcePushFinished(ui64 index) + : Index(index) + { } + + ui64 Index; +}; + +struct TEvSinkPop + : NActors::TEventLocal<TEvSinkPop, TTaskRunnerEvents::ES_SINK_POP> +{ + TEvSinkPop() = default; + TEvSinkPop(ui64 index, i64 size) + : Index(index) + , Size(size) + { } + + ui64 Index; + i64 Size; +}; + +struct TEvSinkPopFinished + : NActors::TEventLocal<TEvSinkPopFinished, TTaskRunnerEvents::ES_SINK_POP_FINISHED> +{ + TEvSinkPopFinished() = default; + TEvSinkPopFinished( + ui64 index, + TMaybe<NDqProto::TCheckpoint>&& checkpoint, + i64 size, + i64 checkpointSize, + bool finished, + bool changed) + : Index(index) + , Checkpoint(std::move(checkpoint)) + , Size(size) + , CheckpointSize(checkpointSize) + , Finished(finished) + , Changed(changed) + { } + + const ui64 Index; + TVector<TString> Strings; + TMaybe<NDqProto::TCheckpoint> Checkpoint; + i64 Size; + i64 CheckpointSize; + bool Finished; + bool Changed; +}; + +} // namespace NTaskRunnerActor + +} // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/task_runner/task_runner_actor.h b/ydb/library/yql/dq/actors/task_runner/task_runner_actor.h index 027a76b09e..7e2ee8c581 100644 --- a/ydb/library/yql/dq/actors/task_runner/task_runner_actor.h +++ b/ydb/library/yql/dq/actors/task_runner/task_runner_actor.h @@ -1,52 +1,52 @@ -#pragma once -#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> -#include <ydb/library/yql/minikql/mkql_node.h> - -#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> +#pragma once +#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> +#include <ydb/library/yql/minikql/mkql_node.h> + +#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <ydb/library/yql/dq/actors/task_runner/events.h> #include <ydb/library/yql/dq/proto/dq_tasks.pb.h> - -namespace NYql::NDq { - -namespace NTaskRunnerActor { - -struct ITaskRunnerActor { - struct ICallbacks { - virtual ~ICallbacks() = default; - virtual void SinkSend( - ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - TMaybe<NDqProto::TCheckpoint>&& checkpoint, - i64 size, - i64 checkpointSize, - bool finished, - bool changed) = 0; - }; - - virtual ~ITaskRunnerActor() = default; - - virtual void SourcePush( - ui64 cookie, - ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - i64 space, - bool finish) = 0; - - virtual void PassAway() = 0; -}; - -struct ITaskRunnerActorFactory { - using TPtr = std::shared_ptr<ITaskRunnerActorFactory>; - - virtual ~ITaskRunnerActorFactory() = default; - - virtual std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( - ITaskRunnerActor::ICallbacks* parent, - const TString& traceId) = 0; -}; - -ITaskRunnerActorFactory::TPtr CreateLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory); - -} // namespace NTaskRunnerActor - -} // namespace NYql::NDq + +namespace NYql::NDq { + +namespace NTaskRunnerActor { + +struct ITaskRunnerActor { + struct ICallbacks { + virtual ~ICallbacks() = default; + virtual void SinkSend( + ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + TMaybe<NDqProto::TCheckpoint>&& checkpoint, + i64 size, + i64 checkpointSize, + bool finished, + bool changed) = 0; + }; + + virtual ~ITaskRunnerActor() = default; + + virtual void SourcePush( + ui64 cookie, + ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + i64 space, + bool finish) = 0; + + virtual void PassAway() = 0; +}; + +struct ITaskRunnerActorFactory { + using TPtr = std::shared_ptr<ITaskRunnerActorFactory>; + + virtual ~ITaskRunnerActorFactory() = default; + + virtual std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( + ITaskRunnerActor::ICallbacks* parent, + const TString& traceId) = 0; +}; + +ITaskRunnerActorFactory::TPtr CreateLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory); + +} // namespace NTaskRunnerActor + +} // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/task_runner/task_runner_actor_local.cpp b/ydb/library/yql/dq/actors/task_runner/task_runner_actor_local.cpp index 9dd805bcb0..e0fd08b5d4 100644 --- a/ydb/library/yql/dq/actors/task_runner/task_runner_actor_local.cpp +++ b/ydb/library/yql/dq/actors/task_runner/task_runner_actor_local.cpp @@ -1,293 +1,293 @@ -#include "task_runner_actor.h" - -#include <ydb/core/protos/services.pb.h> - +#include "task_runner_actor.h" + +#include <ydb/core/protos/services.pb.h> + #include <ydb/library/yql/dq/actors/task_runner/task_runner_actor.h> - + #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> - -#include <library/cpp/actors/core/hfunc.h> - -using namespace NActors; - -namespace NYql::NDq { - -namespace NTaskRunnerActor { - -class TLocalTaskRunnerActor - : public TActor<TLocalTaskRunnerActor> - , public ITaskRunnerActor -{ -public: - static constexpr char ActorName[] = "YQL_DQ_TASK_RUNNER"; - - TLocalTaskRunnerActor(ITaskRunnerActor::ICallbacks* parent, const TTaskRunnerFactory& factory, const TString& traceId) - : TActor<TLocalTaskRunnerActor>(&TLocalTaskRunnerActor::Handler) - , Parent(parent) - , Factory(factory) - , TraceId(traceId) - { } - - ~TLocalTaskRunnerActor() - { } - - STRICT_STFUNC(Handler, { - cFunc(NActors::TEvents::TEvPoison::EventType, TLocalTaskRunnerActor::PassAway); - HFunc(TEvTaskRunnerCreate, OnDqTask); - HFunc(TEvContinueRun, OnContinueRun); - HFunc(TEvPop, OnChannelPop); - HFunc(TEvPush, OnChannelPush); - HFunc(TEvSinkPop, OnSinkPop); - }); - -private: - void PassAway() override { - TActor<TLocalTaskRunnerActor>::PassAway(); - } - - void OnContinueRun(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { - auto guard = TaskRunner->BindAllocator(); - auto inputMap = ev->Get()->AskFreeSpace - ? Inputs - : ev->Get()->InputChannels; - - auto& sourcesMap = Sources; - - try { - guard.GetMutex()->SetLimit(ev->Get()->MemLimit); - auto res = TaskRunner->Run(); - - THashMap<ui32, ui64> inputChannelFreeSpace; - THashMap<ui32, ui64> sourcesFreeSpace; - if (res == ERunStatus::PendingInput) { - for (auto& channelId : inputMap) { - inputChannelFreeSpace[channelId] = TaskRunner->GetInputChannel(channelId)->GetFreeSpace(); - } - - for (auto& index : sourcesMap) { - sourcesFreeSpace[index] = TaskRunner->GetSource(index)->GetFreeSpace(); - } - } - - ctx.Send( - new IEventHandle( - ev->Sender, - SelfId(), - new TEvTaskRunFinished( - res, - std::move(inputChannelFreeSpace), - std::move(sourcesFreeSpace)), - /*flags=*/0, - ev->Cookie)); - } catch (...) { - ctx.Send( - new IEventHandle( - ev->Sender, - SelfId(), - new TEvError(CurrentExceptionMessage(), {}), - /*flags=*/0, - ev->Cookie)); - } - } - - void OnChannelPush(TEvPush::TPtr& ev, const NActors::TActorContext& ctx) { - auto guard = TaskRunner->BindAllocator(); - auto hasData = ev->Get()->HasData; - auto finish = ev->Get()->Finish; - auto askFreeSpace = ev->Get()->AskFreeSpace; - auto channelId = ev->Get()->ChannelId; - auto data = ev->Get()->Data; - - ui64 freeSpace = 0; - if (hasData) { - TaskRunner->GetInputChannel(channelId)->Push(std::move(data)); - if (askFreeSpace) { - freeSpace = TaskRunner->GetInputChannel(channelId)->GetFreeSpace(); - } - } - if (finish) { - TaskRunner->GetInputChannel(channelId)->Finish(); - } - - // run - ctx.Send( - new IEventHandle( - ev->Sender, - SelfId(), - new TEvContinueRun(channelId, freeSpace), - /*flags=*/0, - ev->Cookie)); - } - - void SourcePush( - ui64 cookie, - ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - i64 space, - bool finish) override - { - auto& ctx = NActors::TlsActivationContext; - auto source = TaskRunner->GetSource(index); - source->Push(std::move(batch), space); - if (finish) { - source->Finish(); - } - ctx->Send( - new IEventHandle( - ParentId, - SelfId(), - new TEvSourcePushFinished(index), - /*flags=*/0, - cookie)); - } - - void OnChannelPop(TEvPop::TPtr& ev, const NActors::TActorContext& ctx) { - auto guard = TaskRunner->BindAllocator(); - - auto channelId = ev->Get()->ChannelId; - auto channel = TaskRunner->GetOutputChannel(channelId); - int maxChunks = std::numeric_limits<int>::max(); - auto wasFinished = ev->Get()->WasFinished; - bool changed = false; - bool isFinished = false; - i64 remain = ev->Get()->Size; - ui32 dataSize = 0; - bool hasData = true; - - if (remain == 0) { - // special case to WorkerActor - remain = 5<<20; - maxChunks = 1; - } - - TVector<NDqProto::TData> chunks; - for (;maxChunks && remain > 0 && !isFinished && hasData; maxChunks--, remain -= dataSize) { - NDqProto::TData data; - auto hasData = channel->Pop(data, remain); - dataSize = data.GetRaw().size(); - isFinished = !hasData && channel->IsFinished(); - - changed = changed || hasData || (isFinished != wasFinished); - - if (hasData) { - chunks.emplace_back(std::move(data)); - } - } - - ctx.Send( - new IEventHandle( - ev->Sender, - SelfId(), - new TEvChannelPopFinished( - channelId, - std::move(chunks), - isFinished, - changed), - /*flags=*/0, - ev->Cookie)); - } - - void OnSinkPop(TEvSinkPop::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - auto guard = TaskRunner->BindAllocator(); - auto sink = TaskRunner->GetSink(ev->Get()->Index); - - NKikimr::NMiniKQL::TUnboxedValueVector batch; - NDqProto::TCheckpoint checkpoint; - TMaybe<NDqProto::TCheckpoint> maybeCheckpoint; - i64 size = 0; - i64 checkpointSize = 0; - - if (ev->Get()->Size > 0) { - size = sink->Pop(batch, ev->Get()->Size); - } - bool hasCheckpoint = sink->Pop(checkpoint); - if (hasCheckpoint) { - checkpointSize = checkpoint.ByteSize(); - maybeCheckpoint.ConstructInPlace(std::move(checkpoint)); - } - auto finished = sink->IsFinished(); - bool changed = finished || ev->Get()->Size > 0 || hasCheckpoint; - - Parent->SinkSend(ev->Get()->Index, std::move(batch), std::move(maybeCheckpoint), checkpointSize, size, finished, changed); - } - - void OnDqTask(TEvTaskRunnerCreate::TPtr& ev, const NActors::TActorContext& ctx) { - ParentId = ev->Sender; - TaskRunner = Factory(ev->Get()->Task, [](const TString& message) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, message); - }); - - auto& inputs = ev->Get()->Task.GetInputs(); - for (auto inputId = 0; inputId < inputs.size(); inputId++) { - auto& input = inputs[inputId]; - if (input.HasSource()) { - Sources.emplace(inputId); - } else { - for (auto& channel : input.GetChannels()) { - Inputs.emplace(channel.GetId()); - } - } - } - - auto guard = TaskRunner->BindAllocator(); - try { - TaskRunner->Prepare(ev->Get()->Task, ev->Get()->MemoryLimits, *ev->Get()->ExecCtx, ev->Get()->ParameterProvider); - - auto event = MakeHolder<TEvTaskRunnerCreateFinished>( - TaskRunner->GetSecureParams(), - TaskRunner->GetTaskParams(), - TaskRunner->GetTypeEnv(), - TaskRunner->GetHolderFactory()); - - ctx.Send( - new IEventHandle( - ev->Sender, - SelfId(), - event.Release(), - /*flags=*/0, - ev->Cookie)); - } catch (...) { - ctx.Send( - new IEventHandle(ev->Sender, SelfId(), - new TEvError(CurrentExceptionMessage(), {}), 0, ev->Cookie)); - - } - } - - NActors::TActorId ParentId; - ITaskRunnerActor::ICallbacks* Parent; - TTaskRunnerFactory Factory; - TString TraceId; - THashSet<ui32> Inputs; - THashSet<ui32> Sources; - TIntrusivePtr<NDq::IDqTaskRunner> TaskRunner; -}; - -struct TLocalTaskRunnerActorFactory: public ITaskRunnerActorFactory { - TLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory) - : Factory(factory) - { } - - std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( - ITaskRunnerActor::ICallbacks* parent, - const TString& traceId) override - { - auto* actor = new TLocalTaskRunnerActor(parent, Factory, traceId); - return std::make_tuple( - static_cast<ITaskRunnerActor*>(actor), - static_cast<NActors::IActor*>(actor) - ); - } - - TTaskRunnerFactory Factory; -}; - -ITaskRunnerActorFactory::TPtr CreateLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory) -{ - return ITaskRunnerActorFactory::TPtr(new TLocalTaskRunnerActorFactory(factory)); -} - -} // namespace NTaskRunnerActor - -} // namespace NYql::NDq + +#include <library/cpp/actors/core/hfunc.h> + +using namespace NActors; + +namespace NYql::NDq { + +namespace NTaskRunnerActor { + +class TLocalTaskRunnerActor + : public TActor<TLocalTaskRunnerActor> + , public ITaskRunnerActor +{ +public: + static constexpr char ActorName[] = "YQL_DQ_TASK_RUNNER"; + + TLocalTaskRunnerActor(ITaskRunnerActor::ICallbacks* parent, const TTaskRunnerFactory& factory, const TString& traceId) + : TActor<TLocalTaskRunnerActor>(&TLocalTaskRunnerActor::Handler) + , Parent(parent) + , Factory(factory) + , TraceId(traceId) + { } + + ~TLocalTaskRunnerActor() + { } + + STRICT_STFUNC(Handler, { + cFunc(NActors::TEvents::TEvPoison::EventType, TLocalTaskRunnerActor::PassAway); + HFunc(TEvTaskRunnerCreate, OnDqTask); + HFunc(TEvContinueRun, OnContinueRun); + HFunc(TEvPop, OnChannelPop); + HFunc(TEvPush, OnChannelPush); + HFunc(TEvSinkPop, OnSinkPop); + }); + +private: + void PassAway() override { + TActor<TLocalTaskRunnerActor>::PassAway(); + } + + void OnContinueRun(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { + auto guard = TaskRunner->BindAllocator(); + auto inputMap = ev->Get()->AskFreeSpace + ? Inputs + : ev->Get()->InputChannels; + + auto& sourcesMap = Sources; + + try { + guard.GetMutex()->SetLimit(ev->Get()->MemLimit); + auto res = TaskRunner->Run(); + + THashMap<ui32, ui64> inputChannelFreeSpace; + THashMap<ui32, ui64> sourcesFreeSpace; + if (res == ERunStatus::PendingInput) { + for (auto& channelId : inputMap) { + inputChannelFreeSpace[channelId] = TaskRunner->GetInputChannel(channelId)->GetFreeSpace(); + } + + for (auto& index : sourcesMap) { + sourcesFreeSpace[index] = TaskRunner->GetSource(index)->GetFreeSpace(); + } + } + + ctx.Send( + new IEventHandle( + ev->Sender, + SelfId(), + new TEvTaskRunFinished( + res, + std::move(inputChannelFreeSpace), + std::move(sourcesFreeSpace)), + /*flags=*/0, + ev->Cookie)); + } catch (...) { + ctx.Send( + new IEventHandle( + ev->Sender, + SelfId(), + new TEvError(CurrentExceptionMessage(), {}), + /*flags=*/0, + ev->Cookie)); + } + } + + void OnChannelPush(TEvPush::TPtr& ev, const NActors::TActorContext& ctx) { + auto guard = TaskRunner->BindAllocator(); + auto hasData = ev->Get()->HasData; + auto finish = ev->Get()->Finish; + auto askFreeSpace = ev->Get()->AskFreeSpace; + auto channelId = ev->Get()->ChannelId; + auto data = ev->Get()->Data; + + ui64 freeSpace = 0; + if (hasData) { + TaskRunner->GetInputChannel(channelId)->Push(std::move(data)); + if (askFreeSpace) { + freeSpace = TaskRunner->GetInputChannel(channelId)->GetFreeSpace(); + } + } + if (finish) { + TaskRunner->GetInputChannel(channelId)->Finish(); + } + + // run + ctx.Send( + new IEventHandle( + ev->Sender, + SelfId(), + new TEvContinueRun(channelId, freeSpace), + /*flags=*/0, + ev->Cookie)); + } + + void SourcePush( + ui64 cookie, + ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + i64 space, + bool finish) override + { + auto& ctx = NActors::TlsActivationContext; + auto source = TaskRunner->GetSource(index); + source->Push(std::move(batch), space); + if (finish) { + source->Finish(); + } + ctx->Send( + new IEventHandle( + ParentId, + SelfId(), + new TEvSourcePushFinished(index), + /*flags=*/0, + cookie)); + } + + void OnChannelPop(TEvPop::TPtr& ev, const NActors::TActorContext& ctx) { + auto guard = TaskRunner->BindAllocator(); + + auto channelId = ev->Get()->ChannelId; + auto channel = TaskRunner->GetOutputChannel(channelId); + int maxChunks = std::numeric_limits<int>::max(); + auto wasFinished = ev->Get()->WasFinished; + bool changed = false; + bool isFinished = false; + i64 remain = ev->Get()->Size; + ui32 dataSize = 0; + bool hasData = true; + + if (remain == 0) { + // special case to WorkerActor + remain = 5<<20; + maxChunks = 1; + } + + TVector<NDqProto::TData> chunks; + for (;maxChunks && remain > 0 && !isFinished && hasData; maxChunks--, remain -= dataSize) { + NDqProto::TData data; + auto hasData = channel->Pop(data, remain); + dataSize = data.GetRaw().size(); + isFinished = !hasData && channel->IsFinished(); + + changed = changed || hasData || (isFinished != wasFinished); + + if (hasData) { + chunks.emplace_back(std::move(data)); + } + } + + ctx.Send( + new IEventHandle( + ev->Sender, + SelfId(), + new TEvChannelPopFinished( + channelId, + std::move(chunks), + isFinished, + changed), + /*flags=*/0, + ev->Cookie)); + } + + void OnSinkPop(TEvSinkPop::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto guard = TaskRunner->BindAllocator(); + auto sink = TaskRunner->GetSink(ev->Get()->Index); + + NKikimr::NMiniKQL::TUnboxedValueVector batch; + NDqProto::TCheckpoint checkpoint; + TMaybe<NDqProto::TCheckpoint> maybeCheckpoint; + i64 size = 0; + i64 checkpointSize = 0; + + if (ev->Get()->Size > 0) { + size = sink->Pop(batch, ev->Get()->Size); + } + bool hasCheckpoint = sink->Pop(checkpoint); + if (hasCheckpoint) { + checkpointSize = checkpoint.ByteSize(); + maybeCheckpoint.ConstructInPlace(std::move(checkpoint)); + } + auto finished = sink->IsFinished(); + bool changed = finished || ev->Get()->Size > 0 || hasCheckpoint; + + Parent->SinkSend(ev->Get()->Index, std::move(batch), std::move(maybeCheckpoint), checkpointSize, size, finished, changed); + } + + void OnDqTask(TEvTaskRunnerCreate::TPtr& ev, const NActors::TActorContext& ctx) { + ParentId = ev->Sender; + TaskRunner = Factory(ev->Get()->Task, [](const TString& message) { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PROXY, message); + }); + + auto& inputs = ev->Get()->Task.GetInputs(); + for (auto inputId = 0; inputId < inputs.size(); inputId++) { + auto& input = inputs[inputId]; + if (input.HasSource()) { + Sources.emplace(inputId); + } else { + for (auto& channel : input.GetChannels()) { + Inputs.emplace(channel.GetId()); + } + } + } + + auto guard = TaskRunner->BindAllocator(); + try { + TaskRunner->Prepare(ev->Get()->Task, ev->Get()->MemoryLimits, *ev->Get()->ExecCtx, ev->Get()->ParameterProvider); + + auto event = MakeHolder<TEvTaskRunnerCreateFinished>( + TaskRunner->GetSecureParams(), + TaskRunner->GetTaskParams(), + TaskRunner->GetTypeEnv(), + TaskRunner->GetHolderFactory()); + + ctx.Send( + new IEventHandle( + ev->Sender, + SelfId(), + event.Release(), + /*flags=*/0, + ev->Cookie)); + } catch (...) { + ctx.Send( + new IEventHandle(ev->Sender, SelfId(), + new TEvError(CurrentExceptionMessage(), {}), 0, ev->Cookie)); + + } + } + + NActors::TActorId ParentId; + ITaskRunnerActor::ICallbacks* Parent; + TTaskRunnerFactory Factory; + TString TraceId; + THashSet<ui32> Inputs; + THashSet<ui32> Sources; + TIntrusivePtr<NDq::IDqTaskRunner> TaskRunner; +}; + +struct TLocalTaskRunnerActorFactory: public ITaskRunnerActorFactory { + TLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory) + : Factory(factory) + { } + + std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( + ITaskRunnerActor::ICallbacks* parent, + const TString& traceId) override + { + auto* actor = new TLocalTaskRunnerActor(parent, Factory, traceId); + return std::make_tuple( + static_cast<ITaskRunnerActor*>(actor), + static_cast<NActors::IActor*>(actor) + ); + } + + TTaskRunnerFactory Factory; +}; + +ITaskRunnerActorFactory::TPtr CreateLocalTaskRunnerActorFactory(const TTaskRunnerFactory& factory) +{ + return ITaskRunnerActorFactory::TPtr(new TLocalTaskRunnerActorFactory(factory)); +} + +} // namespace NTaskRunnerActor + +} // namespace NYql::NDq diff --git a/ydb/library/yql/dq/actors/task_runner/ya.make b/ydb/library/yql/dq/actors/task_runner/ya.make index b872367c09..5641ca9d7d 100644 --- a/ydb/library/yql/dq/actors/task_runner/ya.make +++ b/ydb/library/yql/dq/actors/task_runner/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - events.cpp - task_runner_actor_local.cpp -) - -PEERDIR( - library/cpp/actors/core +LIBRARY() + +OWNER(g:yql) + +SRCS( + events.cpp + task_runner_actor_local.cpp +) + +PEERDIR( + library/cpp/actors/core ydb/library/yql/dq/runtime ydb/library/yql/dq/common ydb/library/yql/dq/proto ydb/library/yql/minikql ydb/library/yql/minikql/computation ydb/library/yql/utils/actors - ydb/core/protos -) - -YQL_LAST_ABI_VERSION() - -END() + ydb/core/protos +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/dq/actors/ya.make b/ydb/library/yql/dq/actors/ya.make index 10e2cd5e3b..962f15c886 100644 --- a/ydb/library/yql/dq/actors/ya.make +++ b/ydb/library/yql/dq/actors/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - -GRPC() - -OWNER(g:yql) - +LIBRARY() + +GRPC() + +OWNER(g:yql) + SRCS( dq.cpp dq_events_ids.cpp ) - -PEERDIR( + +PEERDIR( ydb/library/yql/dq/actors/protos -) - -END() - -RECURSE( - compute - protos - task_runner -) +) + +END() + +RECURSE( + compute + protos + task_runner +) diff --git a/ydb/library/yql/dq/common/dq_common.cpp b/ydb/library/yql/dq/common/dq_common.cpp index f42875c4b6..865ed134a8 100644 --- a/ydb/library/yql/dq/common/dq_common.cpp +++ b/ydb/library/yql/dq/common/dq_common.cpp @@ -1,8 +1,8 @@ #include "dq_common.h" - -IOutputStream& operator<<(IOutputStream& stream, const NYql::NDq::TTxId& txId) { + +IOutputStream& operator<<(IOutputStream& stream, const NYql::NDq::TTxId& txId) { std::visit([&stream](auto arg) { - stream << arg; - }, txId); - return stream; -} + stream << arg; + }, txId); + return stream; +} diff --git a/ydb/library/yql/dq/common/dq_common.h b/ydb/library/yql/dq/common/dq_common.h index c94796e396..71cc7be6cb 100644 --- a/ydb/library/yql/dq/common/dq_common.h +++ b/ydb/library/yql/dq/common/dq_common.h @@ -3,30 +3,30 @@ #include <library/cpp/actors/core/event_local.h> #include <library/cpp/actors/core/events.h> -#include <util/generic/variant.h> - +#include <util/generic/variant.h> + namespace NYql::NDq { using TTxId = std::variant<ui64, TString>; - + using TLogFunc = std::function<void(const TString& message)>; template <ui32 TEventSpaceBegin, ui32 TEventSpaceDiff = 0> struct TBaseDqResManEvents { enum { - // New ids must be at the end! + // New ids must be at the end! ES_ALLOCATE_WORKERS_REQUEST = EventSpaceBegin(TEventSpaceBegin) + TEventSpaceDiff, ES_ALLOCATE_WORKERS_RESPONSE, - ES_FREE_WORKERS_NOTIFICATION, - ES_REGISTER_NODE, - ES_REGISTER_NODE_RESPONSE, - - ES_CLUSTER_STATUS, - ES_CLUSTER_STATUS_RESPONSE, + ES_FREE_WORKERS_NOTIFICATION, + ES_REGISTER_NODE, + ES_REGISTER_NODE_RESPONSE, + + ES_CLUSTER_STATUS, + ES_CLUSTER_STATUS_RESPONSE, ES_IS_READY, ES_IS_READY_RESPONSE, - ES_JOB_STOP, - ES_JOB_STOP_RESPONSE, + ES_JOB_STOP, + ES_JOB_STOP_RESPONSE, ES_GET_MASTER, ES_GET_MASTER_RESPONSE, @@ -36,12 +36,12 @@ struct TBaseDqResManEvents { ES_QUERY_STATUS, ES_QUERY_STATUS_RESPONSE, - - ES_ROUTES, - ES_ROUTES_RESPONSE, - - ES_OPERATION_STOP, - ES_OPERATION_STOP_RESPONSE, + + ES_ROUTES, + ES_ROUTES_RESPONSE, + + ES_OPERATION_STOP, + ES_OPERATION_STOP_RESPONSE, }; }; @@ -49,12 +49,12 @@ template <ui32 TEventSpaceBegin, ui32 TEventSpaceDiff = 100> struct TBaseDqExecuterEvents { enum { ES_QUERY = EventSpaceBegin(TEventSpaceBegin) + TEventSpaceDiff, - ES_PROGRAM, + ES_PROGRAM, ES_DQ_TASK, ES_READY_TO_PULL, ES_PULL_RESULT, - ES_RESULT_SET, - ES_DQ_FAILURE, + ES_RESULT_SET, + ES_DQ_FAILURE, ES_GRAPH, ES_GRAPH_FINISHED, ES_GRAPH_EXECUTION_EVENT, @@ -66,17 +66,17 @@ struct TBaseDqDataEvents { enum { ES_PULL_REQUEST = EventSpaceBegin(TEventSpaceBegin) + TEventSpaceDiff, ES_PULL_RESPONSE, - ES_LOCAL_PULL_RESPONSE, - - ES_PING_REQUEST, - ES_PING_RESPONSE, - ES_CONTINUE_RUN, + ES_LOCAL_PULL_RESPONSE, + + ES_PING_REQUEST, + ES_PING_RESPONSE, + ES_CONTINUE_RUN, ES_FULL_RESULT_WRITER_STATUS_REQUEST, ES_FULL_RESULT_WRITER_STATUS_RESPONSE, }; }; - + } // namespace NYql::NDq - -IOutputStream& operator<<(IOutputStream& stream, const NYql::NDq::TTxId& txId); + +IOutputStream& operator<<(IOutputStream& stream, const NYql::NDq::TTxId& txId); diff --git a/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json b/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json index 1aab2a84f5..67627b1b8b 100644 --- a/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json +++ b/ydb/library/yql/dq/expr_nodes/dq_expr_nodes.json @@ -86,7 +86,7 @@ "Builder": {"Generate": "None"}, "Children": [ {"Index": 0, "Name": "Inputs", "Type": "TExprList"}, - {"Index": 1, "Name": "Program", "Type": "TCoLambda"}, + {"Index": 1, "Name": "Program", "Type": "TCoLambda"}, {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}, {"Index": 3, "Name": "Sinks", "Type": "TDqSinksList", "Optional": true} ] @@ -99,7 +99,7 @@ { "Name": "TDqPhyStage", "Base": "TDqStageBase", - "Match": {"Type": "Callable", "Name": "DqPhyStage"} + "Match": {"Type": "Callable", "Name": "DqPhyStage"} }, { "Name": "TDqStageList", diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp index e50b93543e..f60f29dd46 100644 --- a/ydb/library/yql/dq/opt/dq_opt.cpp +++ b/ydb/library/yql/dq/opt/dq_opt.cpp @@ -9,16 +9,16 @@ using namespace NYql::NNodes; namespace NYql::NDq { -TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) { - TDqStageSettings settings{}; +TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) { + TDqStageSettings settings{}; for (const auto& tuple : node.Settings()) { if (const auto name = tuple.Name().Value(); name == IdSettingName) { YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); settings.Id = tuple.Value().Cast<TCoAtom>().Value(); - } else if (name == LogicalIdSettingName) { - YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); - settings.LogicalId = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value()); + } else if (name == LogicalIdSettingName) { + YQL_ENSURE(tuple.Value().Maybe<TCoAtom>()); + settings.LogicalId = FromString<ui64>(tuple.Value().Cast<TCoAtom>().Value()); } else if (name == SinglePartitionSettingName) { settings.SinglePartition = true; } else if (name == IsExternalSetting) { @@ -41,28 +41,28 @@ TDqStageSettings TDqStageSettings::Parse(const TDqStageBase& node) { return settings; } -TDqStageSettings TDqStageSettings::New(const NNodes::TDqStageBase& node) { - auto settings = Parse(node); - - if (!settings.Id) { - settings.Id = CreateGuidAsString(); - } - - return settings; -} - -NNodes::TCoNameValueTupleList TDqStageSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const { +TDqStageSettings TDqStageSettings::New(const NNodes::TDqStageBase& node) { + auto settings = Parse(node); + + if (!settings.Id) { + settings.Id = CreateGuidAsString(); + } + + return settings; +} + +NNodes::TCoNameValueTupleList TDqStageSettings::BuildNode(TExprContext& ctx, TPositionHandle pos) const { TVector<TCoNameValueTuple> settings; - auto logicalId = LogicalId; - if (!logicalId) { - logicalId = ctx.NextUniqueId; - } - - settings.push_back(Build<TCoNameValueTuple>(ctx, pos) - .Name().Build(LogicalIdSettingName) - .Value<TCoAtom>().Build(logicalId) - .Done()); - + auto logicalId = LogicalId; + if (!logicalId) { + logicalId = ctx.NextUniqueId; + } + + settings.push_back(Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(LogicalIdSettingName) + .Value<TCoAtom>().Build(logicalId) + .Done()); + if (Id) { settings.push_back(Build<TCoNameValueTuple>(ctx, pos) .Name().Build(IdSettingName) diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h index 8ef94f903a..5b3146d0f4 100644 --- a/ydb/library/yql/dq/opt/dq_opt.h +++ b/ydb/library/yql/dq/opt/dq_opt.h @@ -8,8 +8,8 @@ namespace NYql::NDq { -struct TDqStageSettings { - static constexpr TStringBuf LogicalIdSettingName = "_logical_id"; +struct TDqStageSettings { + static constexpr TStringBuf LogicalIdSettingName = "_logical_id"; static constexpr TStringBuf IdSettingName = "_id"; static constexpr TStringBuf SinglePartitionSettingName = "_single_partition"; static constexpr TStringBuf IsExternalSetting = "is_external_function"; @@ -17,7 +17,7 @@ struct TDqStageSettings { static constexpr TStringBuf TransformTypeSetting = "transform_type"; static constexpr TStringBuf TransformConcurrencySetting = "concurrency"; - ui64 LogicalId = 0; + ui64 LogicalId = 0; TString Id; bool SinglePartition = false; @@ -26,12 +26,12 @@ struct TDqStageSettings { TString TransformName; ui32 TransformConcurrency = 0; - static TDqStageSettings Parse(const NNodes::TDqStageBase& node); + static TDqStageSettings Parse(const NNodes::TDqStageBase& node); - static TDqStageSettings New(const NNodes::TDqStageBase& node); - - static TDqStageSettings New() { - TDqStageSettings s; + static TDqStageSettings New(const NNodes::TDqStageBase& node); + + static TDqStageSettings New() { + TDqStageSettings s; s.Id = CreateGuidAsString(); return s; } diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp index 1c33b92225..4097b49e24 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp @@ -59,12 +59,12 @@ TExprBase BuildSkipNullKeys(TExprContext& ctx, TPositionHandle pos, TMaybe<TJoinInputDesc> BuildDqJoin(const TCoEquiJoinTuple& joinTuple, const THashMap<TStringBuf, TJoinInputDesc>& inputs, TExprContext& ctx) { - auto options = joinTuple.Options(); - auto linkSettings = GetEquiJoinLinkSettings(options.Ref()); - if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { - return {}; - } - + auto options = joinTuple.Options(); + auto linkSettings = GetEquiJoinLinkSettings(options.Ref()); + if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { + return {}; + } + TMaybe<TJoinInputDesc> left; if (joinTuple.LeftScope().Maybe<TCoAtom>()) { left = inputs.at(joinTuple.LeftScope().Cast<TCoAtom>().Value()); @@ -252,50 +252,50 @@ TDqPhyMapJoin DqMakePhyMapJoin(const TDqJoin& join, const TExprBase& leftInput, } // namespace -// used in yql_dq_recapture.cpp -bool CheckJoinColumns(const TExprBase& node) { - try { - auto equiJoin = node.Cast<TCoEquiJoin>(); +// used in yql_dq_recapture.cpp +bool CheckJoinColumns(const TExprBase& node) { + try { + auto equiJoin = node.Cast<TCoEquiJoin>(); THashMap<TStringBuf, TVector<TStringBuf>> columnsToRename; - THashSet<TStringBuf> columnsToDrop; + THashSet<TStringBuf> columnsToDrop; CollectJoinColumns(equiJoin.Arg(equiJoin.ArgCount() - 1), &columnsToRename, &columnsToDrop); return true; - } catch (...) { - return false; - } -} - -bool CheckJoinTupleLinkSettings(const TCoEquiJoinTuple& joinTuple) { - auto options = joinTuple.Options(); - auto linkSettings = GetEquiJoinLinkSettings(options.Ref()); - if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { - return false; - } - - bool result = true; - if (!joinTuple.LeftScope().Maybe<TCoAtom>()) { - result &= CheckJoinTupleLinkSettings(joinTuple.LeftScope().Cast<TCoEquiJoinTuple>()); - } - if (!result) { - return result; - } - - if (!joinTuple.RightScope().Maybe<TCoAtom>()) { - result &= CheckJoinTupleLinkSettings(joinTuple.RightScope().Cast<TCoEquiJoinTuple>()); - } - return result; -} - -bool CheckJoinLinkSettings(const TExprBase& node) { - if (!node.Maybe<TCoEquiJoin>()) { - return true; - } - auto equiJoin = node.Cast<TCoEquiJoin>(); - YQL_ENSURE(equiJoin.ArgCount() >= 4); - auto joinTuple = equiJoin.Arg(equiJoin.ArgCount() - 2).Cast<TCoEquiJoinTuple>(); - return CheckJoinTupleLinkSettings(joinTuple); -} - + } catch (...) { + return false; + } +} + +bool CheckJoinTupleLinkSettings(const TCoEquiJoinTuple& joinTuple) { + auto options = joinTuple.Options(); + auto linkSettings = GetEquiJoinLinkSettings(options.Ref()); + if (linkSettings.LeftHints.contains("any") || linkSettings.RightHints.contains("any")) { + return false; + } + + bool result = true; + if (!joinTuple.LeftScope().Maybe<TCoAtom>()) { + result &= CheckJoinTupleLinkSettings(joinTuple.LeftScope().Cast<TCoEquiJoinTuple>()); + } + if (!result) { + return result; + } + + if (!joinTuple.RightScope().Maybe<TCoAtom>()) { + result &= CheckJoinTupleLinkSettings(joinTuple.RightScope().Cast<TCoEquiJoinTuple>()); + } + return result; +} + +bool CheckJoinLinkSettings(const TExprBase& node) { + if (!node.Maybe<TCoEquiJoin>()) { + return true; + } + auto equiJoin = node.Cast<TCoEquiJoin>(); + YQL_ENSURE(equiJoin.ArgCount() >= 4); + auto joinTuple = equiJoin.Arg(equiJoin.ArgCount() - 2).Cast<TCoEquiJoinTuple>(); + return CheckJoinTupleLinkSettings(joinTuple); +} + /** * Rewrite `EquiJoin` to a number of `DqJoin` callables. This is done to simplify next step of building * physical stages with join operators. @@ -632,7 +632,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& .Args({leftInputArg, rightInputArg}) .Body(phyJoin.Cast()) .Build() - .Settings(TDqStageSettings().BuildNode(ctx, join.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, join.Pos())) .Done(); newConnection = Build<TDqCnUnionAll>(ctx, join.Pos()) diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp index 1f2e8cbcc6..fed0f09f0f 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp @@ -204,43 +204,43 @@ NNodes::TMaybeNode<NNodes::TExprBase> DqUnorderedInStage(NNodes::TExprBase node, return node; } -NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx) -{ - auto maybeFlatMap = node.Maybe<TCoFlatMapBase>(); - if (!maybeFlatMap) { - return node; - } - auto flatMap = maybeFlatMap.Cast(); - if (!flatMap.Input().Maybe<TCoExtendBase>()) { - return node; - } - - bool hasDqConnection = false;; - auto input = flatMap.Input(); - for (auto child: input.Ref().Children()) { - hasDqConnection |= !!TExprBase{child}.Maybe<TDqConnection>(); - } - - if (!hasDqConnection) { - return node; - } - - const bool ordered = flatMap.Maybe<TCoOrderedFlatMap>() && !input.Maybe<TCoExtend>(); - TExprNode::TListType extendChildren; - for (auto child: input.Ref().Children()) { - extendChildren.push_back(ctx.Builder(child->Pos()) - .Callable(ordered ? TCoOrderedFlatMap::CallableName() : TCoFlatMap::CallableName()) - .Add(0, child) - .Add(1, flatMap.Lambda().Ptr()) - .Seal() - .Build()); - } - TStringBuf extendName = input.Maybe<TCoMerge>() - ? TCoMerge::CallableName() - : (ordered ? TCoOrderedExtend::CallableName() : TCoExtend::CallableName()); - - auto res = ctx.NewCallable(node.Pos(), extendName, std::move(extendChildren)); - return TExprBase(res); +NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx) +{ + auto maybeFlatMap = node.Maybe<TCoFlatMapBase>(); + if (!maybeFlatMap) { + return node; + } + auto flatMap = maybeFlatMap.Cast(); + if (!flatMap.Input().Maybe<TCoExtendBase>()) { + return node; + } + + bool hasDqConnection = false;; + auto input = flatMap.Input(); + for (auto child: input.Ref().Children()) { + hasDqConnection |= !!TExprBase{child}.Maybe<TDqConnection>(); + } + + if (!hasDqConnection) { + return node; + } + + const bool ordered = flatMap.Maybe<TCoOrderedFlatMap>() && !input.Maybe<TCoExtend>(); + TExprNode::TListType extendChildren; + for (auto child: input.Ref().Children()) { + extendChildren.push_back(ctx.Builder(child->Pos()) + .Callable(ordered ? TCoOrderedFlatMap::CallableName() : TCoFlatMap::CallableName()) + .Add(0, child) + .Add(1, flatMap.Lambda().Ptr()) + .Seal() + .Build()); + } + TStringBuf extendName = input.Maybe<TCoMerge>() + ? TCoMerge::CallableName() + : (ordered ? TCoOrderedExtend::CallableName() : TCoExtend::CallableName()); + + auto res = ctx.NewCallable(node.Pos(), extendName, std::move(extendChildren)); + return TExprBase(res); +} + } - -} diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h index cb07d196d9..8e9ed97b06 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.h +++ b/ydb/library/yql/dq/opt/dq_opt_log.h @@ -24,8 +24,8 @@ NNodes::TExprBase DqExpandWindowFunctions(NNodes::TExprBase node, TExprContext& NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprContext& ctx); -NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx); - +NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx); + NNodes::TMaybeNode<NNodes::TExprBase> DqUnorderedInStage(NNodes::TExprBase node, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx); diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp index 514ab8d7fd..cbe871059d 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp @@ -144,7 +144,7 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP .Build() .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); return Build<TDqCnUnionAll>(ctx, node.Pos()) @@ -296,7 +296,7 @@ TMaybeNode<TDqStage> DqPushLambdaToStage(const TDqStage& stage, const TCoAtom& o .Args(newArgs) .Body(ctx.ReplaceNodes(newProgram->TailPtr(), inputArgReplaces)) .Build() - .Settings(TDqStageSettings().BuildNode(ctx, stage.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, stage.Pos())) .Done(); optCtx.RemapNode(stage.Ref(), newStage.Ptr()); @@ -1049,7 +1049,7 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte .KeySelectorLambda(ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref())) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); } @@ -1061,50 +1061,50 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte .Done(); } - -// will generate smth like this -// (let $7 (DqPhyStage '((DqCnUnionAll (TDqOutput $5 '"0"))) (lambda '($13) (FromFlow (Take (Skip (ToFlow $13) (Uint64 '1)) $6))) '())) -// (let $8 (DqPhyStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($14) (FromFlow (Take (ToFlow $14) $6))) '())) -// maybe optimize (Take (Skip ...) ...) ? + +// will generate smth like this +// (let $7 (DqPhyStage '((DqCnUnionAll (TDqOutput $5 '"0"))) (lambda '($13) (FromFlow (Take (Skip (ToFlow $13) (Uint64 '1)) $6))) '())) +// (let $8 (DqPhyStage '((DqCnUnionAll (TDqOutput $7 '"0"))) (lambda '($14) (FromFlow (Take (ToFlow $14) $6))) '())) +// maybe optimize (Take (Skip ...) ...) ? TExprBase DqBuildSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& /* optCtx */, const TParentsMap& parentsMap, bool allowStageMultiUsage) -{ - if (!node.Maybe<TCoSkip>().Input().Maybe<TDqCnUnionAll>()) { - return node; - } - - auto skip = node.Cast<TCoSkip>(); - if (!IsDqPureExpr(skip.Count())) { - return node; - } - - auto dqUnion = skip.Input().Cast<TDqCnUnionAll>(); +{ + if (!node.Maybe<TCoSkip>().Input().Maybe<TDqCnUnionAll>()) { + return node; + } + + auto skip = node.Cast<TCoSkip>(); + if (!IsDqPureExpr(skip.Count())) { + return node; + } + + auto dqUnion = skip.Input().Cast<TDqCnUnionAll>(); if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) { - return node; - } - - auto stage = Build<TDqStage>(ctx, node.Pos()) - .Inputs() - .Add(dqUnion) - .Build() - .Program() - .Args({"stream"}) - .Body<TCoSkip>() - .Input("stream") - .Count(skip.Count()) - .Build() - .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) - .Done(); - - return Build<TDqCnUnionAll>(ctx, node.Pos()) - .Output() - .Stage(stage) - .Index().Build("0") - .Build() - .Done(); -} - + return node; + } + + auto stage = Build<TDqStage>(ctx, node.Pos()) + .Inputs() + .Add(dqUnion) + .Build() + .Program() + .Args({"stream"}) + .Body<TCoSkip>() + .Input("stream") + .Count(skip.Count()) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Done(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage(stage) + .Index().Build("0") + .Build() + .Done(); +} + TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage) { @@ -1150,7 +1150,7 @@ TExprBase DqBuildTakeStage(TExprBase node, TExprContext& ctx, IOptimizationConte .Count(take.Count()) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); return Build<TDqCnUnionAll>(ctx, node.Pos()) @@ -1214,7 +1214,7 @@ TExprBase DqBuildTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationC .Count(take.Count()) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); return Build<TDqCnUnionAll>(ctx, node.Pos()) @@ -1230,15 +1230,15 @@ TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptim return node; } - auto dqUnion = node.Cast<TCoLength>().List().Cast<TDqCnUnionAll>(); + auto dqUnion = node.Cast<TCoLength>().List().Cast<TDqCnUnionAll>(); - auto zero = Build<TCoUint64>(ctx, node.Pos()) + auto zero = Build<TCoUint64>(ctx, node.Pos()) .Literal().Build("0") .Done(); - auto field = BuildAtom("_dq_agg_cnt", node.Pos(), ctx); - - auto combine = Build<TCoCombineByKey>(ctx, node.Pos()) + auto field = BuildAtom("_dq_agg_cnt", node.Pos(), ctx); + + auto combine = Build<TCoCombineByKey>(ctx, node.Pos()) .Input(dqUnion) .PreMapLambda() .Args({"item"}) @@ -1265,23 +1265,23 @@ TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptim .FinishHandlerLambda() .Args({"key", "state"}) .Body<TCoJust>() - .Input<TCoAsStruct>() - .Add<TCoNameValueTuple>() - .Name(field) - .Value("state") - .Build() - .Build() + .Input<TCoAsStruct>() + .Add<TCoNameValueTuple>() + .Name(field) + .Value("state") + .Build() + .Build() .Build() .Build() .Done(); - const auto stub = MakeBool<false>(node.Pos(), ctx); - - auto partition = Build<TCoPartitionsByKeys>(ctx, node.Pos()) + const auto stub = MakeBool<false>(node.Pos(), ctx); + + auto partition = Build<TCoPartitionsByKeys>(ctx, node.Pos()) .Input(combine) .KeySelectorLambda() .Args({"item"}) - .Body(stub) + .Body(stub) .Build() .SortDirections<TCoVoid>() .Build() @@ -1291,50 +1291,50 @@ TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptim .Args({"list"}) .Body<TCoCondense1>() .Input("list") - .InitHandler(BuildIdentityLambda(node.Pos(), ctx)) // take struct from CombineByKey result + .InitHandler(BuildIdentityLambda(node.Pos(), ctx)) // take struct from CombineByKey result .SwitchHandler() .Args({"item", "state"}) - .Body(stub) + .Body(stub) .Build() .UpdateHandler() .Args({"item", "state"}) - .Body<TCoAsStruct>() - .Add<TCoNameValueTuple>() - .Name(field) - .Value<TCoAggrAdd>() - .Left<TCoMember>() - .Struct("state") - .Name(field) - .Build() - .Right<TCoMember>() - .Struct("item") - .Name(field) - .Build() - .Build() - .Build() + .Body<TCoAsStruct>() + .Add<TCoNameValueTuple>() + .Name(field) + .Value<TCoAggrAdd>() + .Left<TCoMember>() + .Struct("state") + .Name(field) + .Build() + .Right<TCoMember>() + .Struct("item") + .Name(field) + .Build() + .Build() + .Build() .Build() .Build() .Build() .Build() .Done(); - auto toOptional = Build<TCoToOptional>(ctx, node.Pos()) - .List(partition) - .Done(); - - auto coalesce = Build<TCoCoalesce>(ctx, node.Pos()) - .Predicate(toOptional) - .Value<TCoAsStruct>() - .Add<TCoNameValueTuple>() - .Name(field) - .Value(zero) - .Build() + auto toOptional = Build<TCoToOptional>(ctx, node.Pos()) + .List(partition) + .Done(); + + auto coalesce = Build<TCoCoalesce>(ctx, node.Pos()) + .Predicate(toOptional) + .Value<TCoAsStruct>() + .Add<TCoNameValueTuple>() + .Name(field) + .Value(zero) + .Build() .Build() - .Done(); - - return Build<TCoMember>(ctx, node.Pos()) - .Struct(coalesce) - .Name(field) + .Done(); + + return Build<TCoMember>(ctx, node.Pos()) + .Struct(coalesce) + .Name(field) .Done(); } @@ -1352,7 +1352,7 @@ TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) { .Input(node) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); return Build<TDqCnUnionAll>(ctx, node.Pos()) @@ -1373,11 +1373,11 @@ TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) { * can be pure expressions not wrapped in DqStage (e.g. ... UNION ALL SELECT 1). */ TExprBase DqBuildExtendStage(TExprBase node, TExprContext& ctx) { - if (!node.Maybe<TCoExtendBase>()) { + if (!node.Maybe<TCoExtendBase>()) { return node; } - auto extend = node.Cast<TCoExtendBase>(); + auto extend = node.Cast<TCoExtendBase>(); TVector<TCoArgument> inputArgs; TVector<TExprBase> inputConns; TVector<TExprBase> extendArgs; @@ -1415,7 +1415,7 @@ TExprBase DqBuildExtendStage(TExprBase node, TExprContext& ctx) { .Add(extendArgs) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Done(); return Build<TDqCnUnionAll>(ctx, node.Pos()) diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.h b/ydb/library/yql/dq/opt/dq_opt_phy.h index 6122d8bb99..70a9d70082 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.h +++ b/ydb/library/yql/dq/opt/dq_opt_phy.h @@ -51,9 +51,9 @@ NNodes::TExprBase DqBuildTopSortStage(NNodes::TExprBase node, TExprContext& ctx, NNodes::TExprBase DqBuildSortStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage = true); -NNodes::TExprBase DqBuildSkipStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, +NNodes::TExprBase DqBuildSkipStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage = true); - + NNodes::TExprBase DqBuildTakeStage(NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TParentsMap& parentsMap, bool allowStageMultiUsage = true); diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp index a19cdabda4..a1c8cf10c2 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp @@ -108,7 +108,7 @@ std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage .Args(newStageArgs) .Body(ctx.ReplaceNodes(std::move(newResult), stageArgsReplaces)) .Build() - .Settings(TDqStageSettings().BuildNode(ctx, stage.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, stage.Pos())) .Done(); YQL_CLOG(TRACE, CoreDq) << "new stage #" << newStage.Ref().UniqueId(); diff --git a/ydb/library/yql/dq/runtime/dq_input_channel.cpp b/ydb/library/yql/dq/runtime/dq_input_channel.cpp index fa16407b03..bd0a541e37 100644 --- a/ydb/library/yql/dq/runtime/dq_input_channel.cpp +++ b/ydb/library/yql/dq/runtime/dq_input_channel.cpp @@ -1,12 +1,12 @@ #include "dq_input_channel.h" #include "dq_input_impl.h" - -namespace NYql::NDq { - + +namespace NYql::NDq { + class TDqInputChannel : public TDqInputImpl<TDqInputChannel, IDqInputChannel> { using TBaseImpl = TDqInputImpl<TDqInputChannel, IDqInputChannel>; friend TBaseImpl; -public: +public: TDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion) @@ -15,20 +15,20 @@ public: , BasicStats(ChannelId) , ProfileStats(collectProfileStats ? &BasicStats : nullptr) , DataSerializer(typeEnv, holderFactory, transportVersion) {} - - ui64 GetChannelId() const override { - return ChannelId; - } - - void Push(NDqProto::TData&& data) override { - YQL_ENSURE(!Finished, "input channel " << ChannelId << " already finished"); - + + ui64 GetChannelId() const override { + return ChannelId; + } + + void Push(NDqProto::TData&& data) override { + YQL_ENSURE(!Finished, "input channel " << ChannelId << " already finished"); + if (Y_UNLIKELY(data.GetRows() == 0)) { return; } const i64 space = data.GetRaw().size(); - + NKikimr::NMiniKQL::TUnboxedValueVector buffer; buffer.reserve(data.GetRows()); @@ -38,28 +38,28 @@ public: ProfileStats->DeserializationTime += (TInstant::Now() - startTime); } else { DataSerializer.Deserialize(data, InputType, buffer); - } - + } + AddBatch(std::move(buffer), space); - } - - const TDqInputChannelStats* GetStats() const override { + } + + const TDqInputChannelStats* GetStats() const override { return &BasicStats; - } - -private: - const ui64 ChannelId; + } + +private: + const ui64 ChannelId; TDqInputChannelStats BasicStats; TDqInputChannelStats* ProfileStats = nullptr; - TDqDataSerializer DataSerializer; -}; - + TDqDataSerializer DataSerializer; +}; + IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory, NDqProto::EDataTransportVersion transportVersion) -{ +{ return new TDqInputChannel(channelId, inputType, maxBufferBytes, collectProfileStats, typeEnv, holderFactory, transportVersion); -} - -} // namespace NYql::NDq +} + +} // namespace NYql::NDq diff --git a/ydb/library/yql/dq/runtime/dq_input_channel.h b/ydb/library/yql/dq/runtime/dq_input_channel.h index 5c770b194c..27020103cc 100644 --- a/ydb/library/yql/dq/runtime/dq_input_channel.h +++ b/ydb/library/yql/dq/runtime/dq_input_channel.h @@ -20,16 +20,16 @@ struct TDqInputChannelStats : TDqInputStats { class IDqInputChannel : public IDqInput { public: - using TPtr = TIntrusivePtr<IDqInputChannel>; + using TPtr = TIntrusivePtr<IDqInputChannel>; - virtual ui64 GetChannelId() const = 0; + virtual ui64 GetChannelId() const = 0; - virtual void Push(NDqProto::TData&& data) = 0; + virtual void Push(NDqProto::TData&& data) = 0; - virtual void Finish() = 0; + virtual void Finish() = 0; - virtual const TDqInputChannelStats* GetStats() const = 0; -}; + virtual const TDqInputChannelStats* GetStats() const = 0; +}; IDqInputChannel::TPtr CreateDqInputChannel(ui64 channelId, NKikimr::NMiniKQL::TType* inputType, ui64 maxBufferBytes, bool collectProfileStats, const NKikimr::NMiniKQL::TTypeEnvironment& typeEnv, diff --git a/ydb/library/yql/dq/runtime/dq_output.h b/ydb/library/yql/dq/runtime/dq_output.h index 39fbeaf4fe..e793ff5231 100644 --- a/ydb/library/yql/dq/runtime/dq_output.h +++ b/ydb/library/yql/dq/runtime/dq_output.h @@ -1,7 +1,7 @@ #pragma once #include <ydb/library/yql/minikql/mkql_node.h> - + #include <util/datetime/base.h> #include <util/generic/ptr.h> @@ -51,8 +51,8 @@ public: virtual bool HasData() const = 0; virtual bool IsFinished() const = 0; - virtual NKikimr::NMiniKQL::TType* GetOutputType() const = 0; - + virtual NKikimr::NMiniKQL::TType* GetOutputType() const = 0; + virtual const TDqOutputStats* GetStats() const = 0; }; diff --git a/ydb/library/yql/dq/runtime/dq_output_channel.cpp b/ydb/library/yql/dq/runtime/dq_output_channel.cpp index 085598d837..211d4ac39a 100644 --- a/ydb/library/yql/dq/runtime/dq_output_channel.cpp +++ b/ydb/library/yql/dq/runtime/dq_output_channel.cpp @@ -283,10 +283,10 @@ public: return rows; } - NKikimr::NMiniKQL::TType* GetOutputType() const override { - return OutputType; - } - + NKikimr::NMiniKQL::TType* GetOutputType() const override { + return OutputType; + } + const TDqOutputChannelStats* GetStats() const override { return &BasicStats; } @@ -763,10 +763,10 @@ public: return rows; } - NKikimr::NMiniKQL::TType* GetOutputType() const override { - return OutputType; - } - + NKikimr::NMiniKQL::TType* GetOutputType() const override { + return OutputType; + } + const TDqOutputChannelStats* GetStats() const override { return &BasicStats; } diff --git a/ydb/library/yql/dq/runtime/dq_sink.cpp b/ydb/library/yql/dq/runtime/dq_sink.cpp index 38ab9a97be..ff86523ef4 100644 --- a/ydb/library/yql/dq/runtime/dq_sink.cpp +++ b/ydb/library/yql/dq/runtime/dq_sink.cpp @@ -137,10 +137,10 @@ public: return true; } - NKikimr::NMiniKQL::TType* GetOutputType() const override { - return OutputType; - } - + NKikimr::NMiniKQL::TType* GetOutputType() const override { + return OutputType; + } + const TDqSinkStats* GetStats() const override { return &BasicStats; } @@ -175,7 +175,7 @@ private: private: const ui64 OutputIndex; const ui64 MaxStoredBytes; - NKikimr::NMiniKQL::TType* const OutputType; + NKikimr::NMiniKQL::TType* const OutputType; ui64 EstimatedStoredBytes = 0; ui64 ValuesPushed = 0; bool Finished = false; diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp index d11814a9dd..10900352c3 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp @@ -558,7 +558,7 @@ public: return TaskHasEffects; } - IDqInputChannel::TPtr GetInputChannel(ui64 channelId) override { + IDqInputChannel::TPtr GetInputChannel(ui64 channelId) override { auto ptr = InputChannels.FindPtr(channelId); YQL_ENSURE(ptr, "task: " << TaskId << " does not have input channelId: " << channelId); return *ptr; diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h index 803211d7ed..a0cc816e91 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h @@ -158,7 +158,7 @@ public: virtual bool HasEffects() const = 0; - virtual IDqInputChannel::TPtr GetInputChannel(ui64 channelId) = 0; + virtual IDqInputChannel::TPtr GetInputChannel(ui64 channelId) = 0; virtual IDqSource::TPtr GetSource(ui64 inputIndex) = 0; virtual IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) = 0; virtual IDqSink::TPtr GetSink(ui64 outputIndex) = 0; diff --git a/ydb/library/yql/dq/runtime/dq_transport.cpp b/ydb/library/yql/dq/runtime/dq_transport.cpp index 2643b8f6bf..93b85b0c6f 100644 --- a/ydb/library/yql/dq/runtime/dq_transport.cpp +++ b/ydb/library/yql/dq/runtime/dq_transport.cpp @@ -184,9 +184,9 @@ void TDqDataSerializer::Deserialize(const NDqProto::TData& data, const TType* it default: YQL_ENSURE(false, "Unsupported TransportVersion"); } -} - - +} + + NDqProto::TData TDqDataSerializer::SerializeParam(const TMkqlValueRef& param, const TTypeEnvironment& typeEnv, const NKikimr::NMiniKQL::THolderFactory& holderFactory) { diff --git a/ydb/library/yql/dq/tasks/dq_connection_builder.h b/ydb/library/yql/dq/tasks/dq_connection_builder.h index 11ee1828aa..2c46061f60 100644 --- a/ydb/library/yql/dq/tasks/dq_connection_builder.h +++ b/ydb/library/yql/dq/tasks/dq_connection_builder.h @@ -192,7 +192,7 @@ void BuildMapChannels(TGraph& graph, const NNodes::TDqPhyStage& stage, ui32 inpu auto& originStageInfo = graph.GetStageInfo(cnMap.Output().Stage()); auto outputIndex = FromString<ui32>(cnMap.Output().Index().Value()); - BuildMapChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, false /*spilling*/, logFunc); + BuildMapChannels(graph, stageInfo, inputIndex, originStageInfo, outputIndex, false /*spilling*/, logFunc); } template <typename TGraph> diff --git a/ydb/library/yql/dq/tasks/dq_tasks_graph.h b/ydb/library/yql/dq/tasks/dq_tasks_graph.h index fc47e7e6bb..e5c772310b 100644 --- a/ydb/library/yql/dq/tasks/dq_tasks_graph.h +++ b/ydb/library/yql/dq/tasks/dq_tasks_graph.h @@ -276,12 +276,12 @@ public: return task; } - void Clear() { - StagesInfo.clear(); - Tasks.clear(); - Channels.clear(); - } - + void Clear() { + StagesInfo.clear(); + Tasks.clear(); + Channels.clear(); + } + private: THashMap<TStageId, TStageInfoType> StagesInfo; TVector<TTaskType> Tasks; diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp index 5d5cf14b20..fcbd3bb9ec 100644 --- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp +++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp @@ -51,7 +51,7 @@ const TTypeAnnotationNode* GetDqConnectionType(const TDqConnection& node, TExprC template <typename TStage> TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) { if (!EnsureMinMaxArgsCount(*stage, 3, 4, ctx)) { - return TStatus::Error; + return TStatus::Error; } auto* inputsTuple = stage->Child(TDqStageBase::idx_Inputs); @@ -196,9 +196,9 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp bool isLeftOptional = IsLeftJoinSideOptional(joinType); auto leftType = ParseJoinInputType(leftRowType, leftLabel, ctx, isLeftOptional); if (leftType.empty() && joinType != "Cross") { - TStringStream str; str << "Cannot parse left join input type: "; - leftRowType.Out(str); - ctx.AddError(TIssue(ctx.GetPosition(pos), str.Str())); + TStringStream str; str << "Cannot parse left join input type: "; + leftRowType.Out(str); + ctx.AddError(TIssue(ctx.GetPosition(pos), str.Str())); return nullptr; } @@ -206,9 +206,9 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp bool isRightOptional = IsRightJoinSideOptional(joinType); auto rightType = ParseJoinInputType(rightRowType, rightLabel, ctx, isRightOptional); if (rightType.empty() && joinType != "Cross") { - TStringStream str; str << "Cannot parse right join input type: "; - rightRowType.Out(str); - ctx.AddError(TIssue(ctx.GetPosition(pos), str.Str())); + TStringStream str; str << "Cannot parse right join input type: "; + rightRowType.Out(str); + ctx.AddError(TIssue(ctx.GetPosition(pos), str.Str())); return nullptr; } @@ -266,16 +266,16 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp } auto addAllMembersFrom = [&ctx](const THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>>& type, - TVector<const TItemExprType*>* result, bool makeOptional = false) + TVector<const TItemExprType*>* result, bool makeOptional = false) { for (const auto& it : type) { for (const auto& it2 : it.second) { auto memberName = FullColumnName(it.first, it2.first); - if (makeOptional && it2.second->GetKind() != ETypeAnnotationKind::Optional) { - result->emplace_back(ctx.MakeType<TItemExprType>(memberName, ctx.MakeType<TOptionalExprType>(it2.second))); - } else { - result->emplace_back(ctx.MakeType<TItemExprType>(memberName, it2.second)); - } + if (makeOptional && it2.second->GetKind() != ETypeAnnotationKind::Optional) { + result->emplace_back(ctx.MakeType<TItemExprType>(memberName, ctx.MakeType<TOptionalExprType>(it2.second))); + } else { + result->emplace_back(ctx.MakeType<TItemExprType>(memberName, it2.second)); + } } } }; @@ -285,7 +285,7 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp addAllMembersFrom(leftType, &resultStructItems, joinType == "Right"); } if (joinType != "LeftOnly" && joinType != "LeftSemi") { - addAllMembersFrom(rightType, &resultStructItems, joinType == "Left"); + addAllMembersFrom(rightType, &resultStructItems, joinType == "Left"); } auto rowType = ctx.MakeType<TStructExprType>(resultStructItems); diff --git a/ydb/library/yql/dq/ya.make b/ydb/library/yql/dq/ya.make index 596e651185..c206ab90ef 100644 --- a/ydb/library/yql/dq/ya.make +++ b/ydb/library/yql/dq/ya.make @@ -1,5 +1,5 @@ RECURSE( - actors + actors common comp_nodes expr_nodes diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp index 7482fdba6c..625a269eaf 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp @@ -582,11 +582,11 @@ NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf, return holderFactory.CreateVariantHolder(UnpackImpl(innerType, buf, topLength, holderFactory).Release(), variantIndex); } - case TType::EKind::Tagged: { - auto taggedType = static_cast<const TTaggedType*>(type); - return UnpackImpl(taggedType->GetBaseType(), buf, topLength, holderFactory); - } - + case TType::EKind::Tagged: { + auto taggedType = static_cast<const TTaggedType*>(type); + return UnpackImpl(taggedType->GetBaseType(), buf, topLength, holderFactory); + } + default: THROW yexception() << "Unsupported type: " << type->GetKindAsStr(); } @@ -974,11 +974,11 @@ bool TValuePacker::HasOptionalFields(const TType* type) { return HasOptionalFields(variantType->GetUnderlyingType()); } - case TType::EKind::Tagged: { - auto taggedType = static_cast<const TTaggedType*>(type); - return HasOptionalFields(taggedType->GetBaseType()); - } - + case TType::EKind::Tagged: { + auto taggedType = static_cast<const TTaggedType*>(type); + return HasOptionalFields(taggedType->GetBaseType()); + } + default: THROW yexception() << "Unsupported type: " << type->GetKindAsStr(); } diff --git a/ydb/library/yql/minikql/mkql_node_cast.cpp b/ydb/library/yql/minikql/mkql_node_cast.cpp index e6aa2c8cef..0a62a7d615 100644 --- a/ydb/library/yql/minikql/mkql_node_cast.cpp +++ b/ydb/library/yql/minikql/mkql_node_cast.cpp @@ -51,7 +51,7 @@ MKQL_AS_TYPE(Resource) MKQL_AS_TYPE(Variant) MKQL_AS_TYPE(Stream) MKQL_AS_TYPE(Flow) -MKQL_AS_TYPE(Tagged) +MKQL_AS_TYPE(Tagged) MKQL_AS_TYPE(Block) MKQL_AS_VALUE(Any, Type) diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 8a8de70ac3..7587e188fa 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -226,13 +226,13 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& return; } - case TType::EKind::Tagged: - { - auto underlyingType = AS_TYPE(TTaggedType, type)->GetBaseType(); - WriteYsonValueImpl(writer, value, underlyingType, structPositions); - return; - } - + case TType::EKind::Tagged: + { + auto underlyingType = AS_TYPE(TTaggedType, type)->GetBaseType(); + WriteYsonValueImpl(writer, value, underlyingType, structPositions); + return; + } + default: YQL_ENSURE(false, "unknown type " << type->GetKindAsStr()); } diff --git a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp index 5875ae7448..4da4d13d03 100644 --- a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp +++ b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp @@ -8,16 +8,16 @@ ui64 TDqIntegrationBase::Partition(const TDqSettings& config, size_t maxPartitio Y_UNUSED(maxPartitions); Y_UNUSED(node); Y_UNUSED(partitions); - Y_UNUSED(clusterName); - Y_UNUSED(ctx); + Y_UNUSED(clusterName); + Y_UNUSED(ctx); Y_UNUSED(canFallback); - return 0; + return 0; } TMaybe<ui64> TDqIntegrationBase::CanRead(const TDqSettings&, const TExprNode& read, TExprContext& ctx, bool skipIssues) { Y_UNUSED(read); - Y_UNUSED(ctx); - Y_UNUSED(skipIssues); + Y_UNUSED(ctx); + Y_UNUSED(skipIssues); return Nothing(); } diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp index dd759e20f5..1fb7bdcbba 100644 --- a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp +++ b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp @@ -15,9 +15,9 @@ namespace { ////////////////////////////////////////////////////////////////////////////// class TCountersPhotographer: public NMonitoring::ICountableConsumer { public: - TCountersPhotographer(NProto::TCounterGroup* groupProto, bool invalidate) + TCountersPhotographer(NProto::TCounterGroup* groupProto, bool invalidate) : HasAnyCounters_(false) - , Invalidate_(invalidate) + , Invalidate_(invalidate) { GroupsProto_.push(groupProto); } @@ -37,9 +37,9 @@ private: counterProto->SetDerivative(counter->ForDerivative()); auto counterVal = counter->Val(); counterProto->SetValue(counterVal); - if (Invalidate_) { + if (Invalidate_) { const_cast<TSensorCounter*>(counter)->Sub(counterVal); - } + } auto* label = counterProto->MutableLabel(); label->SetName(labelName); @@ -47,23 +47,23 @@ private: } void OnHistogram(const TString& labelName, const TString& labelValue, NMonitoring::IHistogramSnapshotPtr snapshot, bool) override { - if (Invalidate_) { - return; - } - auto* counterProto = GroupsProto_.top()->AddCounters(); - auto* label = counterProto->MutableLabel(); - label->SetName(labelName); - label->SetValue(labelValue); - - auto bucketsCount = snapshot->Count(); - for (ui32 i = 0; i < bucketsCount; i += 1) { - auto upperBound = snapshot->UpperBound(i); - auto value = snapshot->Value(i); - - auto* bucket = counterProto->AddBucket(); - bucket->SetUpperBound(upperBound); - bucket->SetValue(value); - } + if (Invalidate_) { + return; + } + auto* counterProto = GroupsProto_.top()->AddCounters(); + auto* label = counterProto->MutableLabel(); + label->SetName(labelName); + label->SetValue(labelValue); + + auto bucketsCount = snapshot->Count(); + for (ui32 i = 0; i < bucketsCount; i += 1) { + auto upperBound = snapshot->UpperBound(i); + auto value = snapshot->Value(i); + + auto* bucket = counterProto->AddBucket(); + bucket->SetUpperBound(upperBound); + bucket->SetValue(value); + } } void OnGroupBegin( @@ -90,7 +90,7 @@ private: private: TStack<NProto::TCounterGroup*> GroupsProto_; bool HasAnyCounters_; - bool Invalidate_; + bool Invalidate_; }; ////////////////////////////////////////////////////////////////////////////// @@ -150,31 +150,31 @@ public: } } - void AddCounter( - const TString& labelName, - const TString& labelValue, - i64 value, - bool derivative) override - { - // total aggregate counter - auto totalCnt = GetCounter(labelName, labelValue, nullptr, derivative); - if (totalCnt) { - totalCnt->Add(value); - } - - if (UserName_) { - // per user counter - auto userCnt = GetCounter(labelName, labelValue, UserName_.Get(), - derivative); - if (userCnt) { - userCnt->Add(value); - } - } - } - + void AddCounter( + const TString& labelName, + const TString& labelValue, + i64 value, + bool derivative) override + { + // total aggregate counter + auto totalCnt = GetCounter(labelName, labelValue, nullptr, derivative); + if (totalCnt) { + totalCnt->Add(value); + } + + if (UserName_) { + // per user counter + auto userCnt = GetCounter(labelName, labelValue, UserName_.Get(), + derivative); + if (userCnt) { + userCnt->Add(value); + } + } + } + bool TakeSnapshot(NProto::TMetricsRegistrySnapshot* snapshot) const override { bool hasRootGroupBefore = snapshot->HasRootGroup(); - TCountersPhotographer photographer(snapshot->MutableRootGroup(), snapshot->GetDontIncrement() == false); + TCountersPhotographer photographer(snapshot->MutableRootGroup(), snapshot->GetDontIncrement() == false); Sensors_->Accept(TString(), TString(), photographer); if (!photographer.HasAnyCounters() && !hasRootGroupBefore) { // remove prematurely allocated group @@ -190,9 +190,9 @@ public: ? GetSensorsRootGroup().Get() : Sensors_.Get(), snapshot.GetRootGroup(), - snapshot.HasDontIncrement() - ? snapshot.GetDontIncrement() - : false); + snapshot.HasDontIncrement() + ? snapshot.GetDontIncrement() + : false); } IMetricsRegistryPtr Personalized(const TString& userName) const override { @@ -225,57 +225,57 @@ private: } void MergeFromGroupProto( - TSensorsGroup* group, const NProto::TCounterGroup& groupProto, bool asIs) + TSensorsGroup* group, const NProto::TCounterGroup& groupProto, bool asIs) { for (const auto& counterProto: groupProto.GetCounters()) { const auto& label = counterProto.GetLabel(); - - if (!counterProto.GetBucket().empty()) { - NMonitoring::TBucketBounds bounds; - auto histSnapshot = NMonitoring::TExplicitHistogramSnapshot::New(counterProto.GetBucket().size()); - bounds.reserve(counterProto.GetBucket().size()); - int i = 0; - for (const auto& b : counterProto.GetBucket()) { - if (i < counterProto.GetBucket().size() - 1) { - // skip inf - bounds.push_back(b.GetUpperBound()); - } - (*histSnapshot)[i].first = b.GetUpperBound(); - (*histSnapshot)[i].second = b.GetValue(); - i += 1; - } - - auto collector = NMonitoring::ExplicitHistogram(bounds).Release(); - auto histogram = group->GetNamedHistogram( - label.GetName(), label.GetValue(), + + if (!counterProto.GetBucket().empty()) { + NMonitoring::TBucketBounds bounds; + auto histSnapshot = NMonitoring::TExplicitHistogramSnapshot::New(counterProto.GetBucket().size()); + bounds.reserve(counterProto.GetBucket().size()); + int i = 0; + for (const auto& b : counterProto.GetBucket()) { + if (i < counterProto.GetBucket().size() - 1) { + // skip inf + bounds.push_back(b.GetUpperBound()); + } + (*histSnapshot)[i].first = b.GetUpperBound(); + (*histSnapshot)[i].second = b.GetValue(); + i += 1; + } + + auto collector = NMonitoring::ExplicitHistogram(bounds).Release(); + auto histogram = group->GetNamedHistogram( + label.GetName(), label.GetValue(), THolder(collector)); - Histograms.insert(std::make_pair(histogram, collector)); - Histograms[histogram]->Collect(*histSnapshot); - } else { - auto counter = group->GetNamedCounter( - label.GetName(), label.GetValue(), - counterProto.GetDerivative()); - if (asIs) { - *counter = counterProto.GetValue(); - } else { - *counter += counterProto.GetValue(); - } - } + Histograms.insert(std::make_pair(histogram, collector)); + Histograms[histogram]->Collect(*histSnapshot); + } else { + auto counter = group->GetNamedCounter( + label.GetName(), label.GetValue(), + counterProto.GetDerivative()); + if (asIs) { + *counter = counterProto.GetValue(); + } else { + *counter += counterProto.GetValue(); + } + } } for (const auto& subGroupProto: groupProto.GetGroups()) { const auto& label = subGroupProto.GetLabel(); auto subGroup = group->GetSubgroup( label.GetName(), label.GetValue()); - MergeFromGroupProto(subGroup.Get(), subGroupProto, asIs); + MergeFromGroupProto(subGroup.Get(), subGroupProto, asIs); } } private: TSensorsGroupPtr Sensors_; const TMaybe<TString> UserName_; - - THashMap<NMonitoring::THistogramPtr, NMonitoring::IHistogramCollector*> Histograms; + + THashMap<NMonitoring::THistogramPtr, NMonitoring::IHistogramCollector*> Histograms; }; } // namespace diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.h b/ydb/library/yql/providers/common/metrics/metrics_registry.h index fb3fcee5da..d344ba9dd9 100644 --- a/ydb/library/yql/providers/common/metrics/metrics_registry.h +++ b/ydb/library/yql/providers/common/metrics/metrics_registry.h @@ -32,12 +32,12 @@ struct IMetricsRegistry: public TThrRefBase { const TString& labelValue, bool derivative = true) = 0; - virtual void AddCounter( - const TString& labelName, - const TString& labelValue, - i64 value, - bool derivative = true) = 0; - + virtual void AddCounter( + const TString& labelName, + const TString& labelValue, + i64 value, + bool derivative = true) = 0; + // will invalidate all counters virtual bool TakeSnapshot( NProto::TMetricsRegistrySnapshot* snapshot) const = 0; diff --git a/ydb/library/yql/providers/common/metrics/protos/metrics_registry.proto b/ydb/library/yql/providers/common/metrics/protos/metrics_registry.proto index 61213ffbc7..0714446258 100644 --- a/ydb/library/yql/providers/common/metrics/protos/metrics_registry.proto +++ b/ydb/library/yql/providers/common/metrics/protos/metrics_registry.proto @@ -6,16 +6,16 @@ message TLabel { optional string Value = 2; } -message TBucket { - optional double UpperBound = 1; - optional int64 Value = 2; -} - +message TBucket { + optional double UpperBound = 1; + optional int64 Value = 2; +} + message TCounter { optional TLabel Label = 1; optional int64 Value = 2; optional bool Derivative = 3; - repeated TBucket Bucket = 4; + repeated TBucket Bucket = 4; } message TCounterGroup { @@ -26,6 +26,6 @@ message TCounterGroup { message TMetricsRegistrySnapshot { optional TCounterGroup RootGroup = 1; - optional bool DontIncrement = 2; + optional bool DontIncrement = 2; optional bool MergeToRoot = 3; } diff --git a/ydb/library/yql/providers/common/metrics/protos/ya.make b/ydb/library/yql/providers/common/metrics/protos/ya.make index 4a55b6082e..f8edeea315 100644 --- a/ydb/library/yql/providers/common/metrics/protos/ya.make +++ b/ydb/library/yql/providers/common/metrics/protos/ya.make @@ -1,13 +1,13 @@ -PROTO_LIBRARY() - -OWNER( - g:yql -) - -SRCS( - metrics_registry.proto -) - -EXCLUDE_TAGS(GO_PROTO) - -END() +PROTO_LIBRARY() + +OWNER( + g:yql +) + +SRCS( + metrics_registry.proto +) + +EXCLUDE_TAGS(GO_PROTO) + +END() diff --git a/ydb/library/yql/providers/common/metrics/ya.make b/ydb/library/yql/providers/common/metrics/ya.make index 5b5dfbf32c..19f8832060 100644 --- a/ydb/library/yql/providers/common/metrics/ya.make +++ b/ydb/library/yql/providers/common/metrics/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - g:yql -) - -SRCS( - metrics_registry.cpp - sensors_group.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + g:yql +) + +SRCS( + metrics_registry.cpp + sensors_group.cpp +) + +PEERDIR( library/cpp/logger/global library/cpp/monlib/dynamic_counters ydb/library/yql/providers/common/metrics/protos -) - -END() +) + +END() diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index c64f6e9f56..8561fe2536 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -391,22 +391,22 @@ message TMysqlGatewayConfig { } -/////////////////////////////// Dq ///////////////////////////////////// -message TDqGatewayConfig { - message TDefaultAutoByHourPercentage { - required uint32 Hour = 1; - required uint32 Percentage = 2; - } - - optional uint32 DefaultAutoPercentage = 1 [default = 0]; // Probability of 'DqEngine="auto"' - repeated TDefaultAutoByHourPercentage DefaultAutoByHour = 2; - repeated string NoDefaultAutoForUsers = 3; - - repeated string DefaultAnalyzeQueryForUsers = 4; - - repeated TAttr DefaultSettings = 102; -} - +/////////////////////////////// Dq ///////////////////////////////////// +message TDqGatewayConfig { + message TDefaultAutoByHourPercentage { + required uint32 Hour = 1; + required uint32 Percentage = 2; + } + + optional uint32 DefaultAutoPercentage = 1 [default = 0]; // Probability of 'DqEngine="auto"' + repeated TDefaultAutoByHourPercentage DefaultAutoByHour = 2; + repeated string NoDefaultAutoForUsers = 3; + + repeated string DefaultAnalyzeQueryForUsers = 4; + + repeated TAttr DefaultSettings = 102; +} + /////////////////////////////// Yql Core /////////////////////////////// message TCoreAttr { @@ -448,7 +448,7 @@ message TGatewaysConfig { optional TYqlCoreConfig YqlCore = 10; optional TPostgresqlGatewayConfig Postgresql = 11; optional TSqlCoreConfig SqlCore = 12; - optional TDqGatewayConfig Dq = 13; + optional TDqGatewayConfig Dq = 13; optional TMysqlGatewayConfig Mysql = 14; optional TYdbGatewayConfig Ydb = 15; optional TPqGatewayConfig Pq = 16; diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp index f2aeb0d676..8de3d57fd3 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp @@ -143,30 +143,30 @@ TExprNode::TPtr TDataProviderBase::RewriteIO(const TExprNode::TPtr& node, TExprC void TDataProviderBase::PostRewriteIO() { } -void TDataProviderBase::Reset() { - GetConfigurationTransformer().Rewind(); - GetIODiscoveryTransformer().Rewind(); - GetEpochsTransformer().Rewind(); - GetIntentDeterminationTransformer().Rewind(); - for (auto flag : {true, false}) { - GetTypeAnnotationTransformer(flag).Rewind(); - } - for (auto flag1 : {true, false}) { - for (auto flag2 : {true, false}) { - GetConstraintTransformer(flag1, flag2).Rewind(); - } - } - GetRecaptureOptProposalTransformer().Rewind(); - GetLogicalOptProposalTransformer().Rewind(); - GetPhysicalOptProposalTransformer().Rewind(); - GetPhysicalFinalizingTransformer().Rewind(); - GetLoadTableMetadataTransformer().Rewind(); - GetCallableExecutionTransformer().Rewind(); - GetFinalizingTransformer().Rewind(); - GetPlanInfoTransformer().Rewind(); +void TDataProviderBase::Reset() { + GetConfigurationTransformer().Rewind(); + GetIODiscoveryTransformer().Rewind(); + GetEpochsTransformer().Rewind(); + GetIntentDeterminationTransformer().Rewind(); + for (auto flag : {true, false}) { + GetTypeAnnotationTransformer(flag).Rewind(); + } + for (auto flag1 : {true, false}) { + for (auto flag2 : {true, false}) { + GetConstraintTransformer(flag1, flag2).Rewind(); + } + } + GetRecaptureOptProposalTransformer().Rewind(); + GetLogicalOptProposalTransformer().Rewind(); + GetPhysicalOptProposalTransformer().Rewind(); + GetPhysicalFinalizingTransformer().Rewind(); + GetLoadTableMetadataTransformer().Rewind(); + GetCallableExecutionTransformer().Rewind(); + GetFinalizingTransformer().Rewind(); + GetPlanInfoTransformer().Rewind(); GetTrackableNodeProcessor().GetCleanupTransformer().Rewind(); -} - +} + IGraphTransformer& TDataProviderBase::GetRecaptureOptProposalTransformer() { return NullTransformer_; } diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h index c80d0ca46e..242a39877e 100644 --- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h +++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h @@ -55,7 +55,7 @@ public: IGraphTransformer& GetConstraintTransformer(bool instantOnly, bool subGraph) override; TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override; void PostRewriteIO() override; - void Reset() override; + void Reset() override; IGraphTransformer& GetRecaptureOptProposalTransformer() override; IGraphTransformer& GetLogicalOptProposalTransformer() override; IGraphTransformer& GetPhysicalOptProposalTransformer() override; diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp index 001edf9dec..aa782ee93f 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.cpp +++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp @@ -940,112 +940,112 @@ double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx) { } void WriteStatistics(NYson::TYsonWriter& writer, bool totalOnly, const THashMap<ui32, TOperationStatistics>& statistics) { - if (statistics.empty()) { - return; - } - - THashMap<TString, std::tuple<i64, i64, i64, TMaybe<i64>>> total; // sum, count, max, min - - writer.OnBeginMap(); - - if (totalOnly) { - for (const auto& opStatistics : statistics) { - for (auto& el : opStatistics.second.Entries) { - if (el.Value) { - continue; - } - - auto& totalEntry = total[el.Name]; - if (auto val = el.Sum) { - std::get<0>(totalEntry) += *val; - } - if (auto val = el.Count) { - std::get<1>(totalEntry) += *val; - } - if (auto val = el.Max) { - std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); - } - if (auto val = el.Min) { - std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); - } - } - } - } - else { - for (const auto& opStatistics : statistics) { - writer.OnKeyedItem(ToString(opStatistics.first)); - writer.OnBeginMap(); - for (auto& el : opStatistics.second.Entries) { - writer.OnKeyedItem(el.Name); - if (el.Value) { - writer.OnStringScalar(*el.Value); - continue; - } - - auto& totalEntry = total[el.Name]; - writer.OnBeginMap(); - if (auto val = el.Sum) { - writer.OnKeyedItem("sum"); - writer.OnInt64Scalar(*val); - std::get<0>(totalEntry) += *val; - } - if (auto val = el.Count) { - writer.OnKeyedItem("count"); - writer.OnInt64Scalar(*val); - std::get<1>(totalEntry) += *val; - } - if (auto val = el.Avg) { - writer.OnKeyedItem("avg"); - writer.OnInt64Scalar(*val); - } - if (auto val = el.Max) { - writer.OnKeyedItem("max"); - writer.OnInt64Scalar(*val); - std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); - } - if (auto val = el.Min) { - writer.OnKeyedItem("min"); - writer.OnInt64Scalar(*val); - std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); - } - writer.OnEndMap(); - } - writer.OnEndMap(); - } - } - - TVector<TString> statKeys; - std::transform(total.cbegin(), total.cend(), std::back_inserter(statKeys), [](const decltype(total)::value_type& v) { return v.first; }); - std::sort(statKeys.begin(), statKeys.end()); - - writer.OnKeyedItem("total"); - writer.OnBeginMap(); - for (auto& key: statKeys) { - auto& totalEntry = total[key]; - writer.OnKeyedItem(key); - writer.OnBeginMap(); - - writer.OnKeyedItem("sum"); - writer.OnInt64Scalar(std::get<0>(totalEntry)); - - writer.OnKeyedItem("count"); - writer.OnInt64Scalar(std::get<1>(totalEntry)); - - writer.OnKeyedItem("avg"); - writer.OnInt64Scalar(std::get<1>(totalEntry) ? (std::get<0>(totalEntry) / std::get<1>(totalEntry)) : 0l); - - writer.OnKeyedItem("max"); - writer.OnInt64Scalar(std::get<2>(totalEntry)); - - writer.OnKeyedItem("min"); - writer.OnInt64Scalar(std::get<3>(totalEntry).GetOrElse(0)); - - writer.OnEndMap(); - } - writer.OnEndMap(); // total - - writer.OnEndMap(); -} - + if (statistics.empty()) { + return; + } + + THashMap<TString, std::tuple<i64, i64, i64, TMaybe<i64>>> total; // sum, count, max, min + + writer.OnBeginMap(); + + if (totalOnly) { + for (const auto& opStatistics : statistics) { + for (auto& el : opStatistics.second.Entries) { + if (el.Value) { + continue; + } + + auto& totalEntry = total[el.Name]; + if (auto val = el.Sum) { + std::get<0>(totalEntry) += *val; + } + if (auto val = el.Count) { + std::get<1>(totalEntry) += *val; + } + if (auto val = el.Max) { + std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); + } + if (auto val = el.Min) { + std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); + } + } + } + } + else { + for (const auto& opStatistics : statistics) { + writer.OnKeyedItem(ToString(opStatistics.first)); + writer.OnBeginMap(); + for (auto& el : opStatistics.second.Entries) { + writer.OnKeyedItem(el.Name); + if (el.Value) { + writer.OnStringScalar(*el.Value); + continue; + } + + auto& totalEntry = total[el.Name]; + writer.OnBeginMap(); + if (auto val = el.Sum) { + writer.OnKeyedItem("sum"); + writer.OnInt64Scalar(*val); + std::get<0>(totalEntry) += *val; + } + if (auto val = el.Count) { + writer.OnKeyedItem("count"); + writer.OnInt64Scalar(*val); + std::get<1>(totalEntry) += *val; + } + if (auto val = el.Avg) { + writer.OnKeyedItem("avg"); + writer.OnInt64Scalar(*val); + } + if (auto val = el.Max) { + writer.OnKeyedItem("max"); + writer.OnInt64Scalar(*val); + std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); + } + if (auto val = el.Min) { + writer.OnKeyedItem("min"); + writer.OnInt64Scalar(*val); + std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); + } + writer.OnEndMap(); + } + writer.OnEndMap(); + } + } + + TVector<TString> statKeys; + std::transform(total.cbegin(), total.cend(), std::back_inserter(statKeys), [](const decltype(total)::value_type& v) { return v.first; }); + std::sort(statKeys.begin(), statKeys.end()); + + writer.OnKeyedItem("total"); + writer.OnBeginMap(); + for (auto& key: statKeys) { + auto& totalEntry = total[key]; + writer.OnKeyedItem(key); + writer.OnBeginMap(); + + writer.OnKeyedItem("sum"); + writer.OnInt64Scalar(std::get<0>(totalEntry)); + + writer.OnKeyedItem("count"); + writer.OnInt64Scalar(std::get<1>(totalEntry)); + + writer.OnKeyedItem("avg"); + writer.OnInt64Scalar(std::get<1>(totalEntry) ? (std::get<0>(totalEntry) / std::get<1>(totalEntry)) : 0l); + + writer.OnKeyedItem("max"); + writer.OnInt64Scalar(std::get<2>(totalEntry)); + + writer.OnKeyedItem("min"); + writer.OnInt64Scalar(std::get<3>(totalEntry).GetOrElse(0)); + + writer.OnEndMap(); + } + writer.OnEndMap(); // total + + writer.OnEndMap(); +} + } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h index 185befb744..9e80cfd135 100644 --- a/ydb/library/yql/providers/common/provider/yql_provider.h +++ b/ydb/library/yql/providers/common/provider/yql_provider.h @@ -28,7 +28,7 @@ namespace NKikimr { namespace NYql { struct TTypeAnnotationContext; -struct TOperationStatistics; +struct TOperationStatistics; namespace NCommon { @@ -119,6 +119,6 @@ void WriteStreams(NYson::TYsonWriter& writer, TStringBuf name, const NNodes::TCo double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx); void WriteStatistics(NYson::TYsonWriter& writer, bool totalOnly, const THashMap<ui32, TOperationStatistics>& statistics); - + } // namespace NCommon } // namespace NYql diff --git a/ydb/library/yql/providers/common/ya.make b/ydb/library/yql/providers/common/ya.make index 00fb1de719..66778b0bcf 100644 --- a/ydb/library/yql/providers/common/ya.make +++ b/ydb/library/yql/providers/common/ya.make @@ -4,7 +4,7 @@ RECURSE( dq gateway http_gateway - metrics + metrics mkql comp_nodes proto diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp index 3d7ee5014d..6952e96cf0 100644 --- a/ydb/library/yql/providers/config/yql_config_provider.cpp +++ b/ydb/library/yql/providers/config/yql_config_provider.cpp @@ -663,26 +663,26 @@ namespace { return false; } } - else if (name == "DqEngine") { - if (args.size() != 1) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size())); - return false; - } - + else if (name == "DqEngine") { + if (args.size() != 1) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size())); + return false; + } + auto arg = TString{args[0]}; - if (Find(Types.AvailablePureResultDataSources, DqProviderName) == Types.AvailablePureResultDataSources.end() || arg == "disable") { - ; // reserved - } else if (arg == "auto") { + if (Find(Types.AvailablePureResultDataSources, DqProviderName) == Types.AvailablePureResultDataSources.end() || arg == "disable") { + ; // reserved + } else if (arg == "auto") { Types.PureResultDataSource = DqProviderName; - Types.ForceDq = false; - } else if (arg == "force") { + Types.ForceDq = false; + } else if (arg == "force") { Types.PureResultDataSource = DqProviderName; - Types.ForceDq = true; - } else { - ctx.AddError(TIssue(pos, TStringBuilder() << "Expected `disable|auto|force', but got: " << args[0])); - return false; - } - } + Types.ForceDq = true; + } else { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected `disable|auto|force', but got: " << args[0])); + return false; + } + } else if (name == "IssueCountLimit") { if (args.size() != 1) { ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size())); @@ -709,15 +709,15 @@ namespace { } Types.StrictTableProps = false; } - else if (name == "GeobaseDownloadUrl") { - if (args.size() != 1) { - ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size())); - return false; - } - Types.UserDataStorageCrutches - .emplace( - TUserDataKey::File(TStringBuf("/home/geodata6.bin")), - TUserDataBlock{EUserDataType::URL, {}, TString(args[0]), {}, {}}).first->second.Usage.Set(EUserDataBlockUsage::Path); + else if (name == "GeobaseDownloadUrl") { + if (args.size() != 1) { + ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size())); + return false; + } + Types.UserDataStorageCrutches + .emplace( + TUserDataKey::File(TStringBuf("/home/geodata6.bin")), + TUserDataBlock{EUserDataType::URL, {}, TString(args[0]), {}, {}}).first->second.Usage.Set(EUserDataBlockUsage::Path); } else if (name == "JsonQueryReturnsJsonDocument" || name == "DisableJsonQueryReturnsJsonDocument") { if (args.size() != 0) { diff --git a/ydb/library/yql/providers/dq/actors/actor_helpers.h b/ydb/library/yql/providers/dq/actors/actor_helpers.h index a343cb7acf..b92603ab0f 100644 --- a/ydb/library/yql/providers/dq/actors/actor_helpers.h +++ b/ydb/library/yql/providers/dq/actors/actor_helpers.h @@ -1,15 +1,15 @@ -#pragma once - +#pragma once + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/interconnect.h> -#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/interconnect.h> +#include <library/cpp/actors/core/hfunc.h> #include <ydb/library/yql/utils/actors/rich_actor.h> - + #include <ydb/library/yql/utils/log/log.h> -namespace NYql { - +namespace NYql { + enum EExecutorPoolType { Main, FullResultWriter, @@ -18,67 +18,67 @@ enum EExecutorPoolType { }; template <typename EventType> -struct TRichActorFutureCallback : public TRichActor<TRichActorFutureCallback<EventType>> { - using TCallback = std::function<void(TAutoPtr<NActors::TEventHandle<EventType>>&)>; - using TFailure = std::function<void(void)>; - using TBase = TRichActor<TRichActorFutureCallback<EventType>>; - +struct TRichActorFutureCallback : public TRichActor<TRichActorFutureCallback<EventType>> { + using TCallback = std::function<void(TAutoPtr<NActors::TEventHandle<EventType>>&)>; + using TFailure = std::function<void(void)>; + using TBase = TRichActor<TRichActorFutureCallback<EventType>>; + static constexpr char ActorName[] = "YQL_DQ_ACTOR_FUTURE_CALLBACK"; - - TRichActorFutureCallback(TCallback&& callback, TFailure&& failure, TDuration timeout) - : TBase(&TRichActorFutureCallback::StateWaitForEvent) - , Callback(std::move(callback)) - , Failure(std::move(failure)) - , Timeout(timeout) - { } - -private: - const TCallback Callback; - const TFailure Failure; - const TDuration Timeout; - bool TimerStarted = false; - NActors::TSchedulerCookieHolder TimerCookieHolder; - - STRICT_STFUNC(StateWaitForEvent, - HFunc(EventType, Handle) - cFunc(NActors::TEvents::TEvBootstrap::EventType, OnFailure) - hFunc(NActors::TEvInterconnect::TEvNodeConnected, [this] (NActors::TEvInterconnect::TEvNodeConnected::TPtr& ev) mutable { - this->Subscribe(ev->Get()->NodeId); - }) - hFunc(NActors::TEvInterconnect::TEvNodeDisconnected, [this] (NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) mutable { - this->Unsubscribe(ev->Get()->NodeId); - TimerStarted = true; - OnFailure(); - }) - hFunc(NActors::TEvents::TEvUndelivered, [this] (NActors::TEvents::TEvUndelivered::TPtr& ev) mutable { - this->Unsubscribe(ev->Sender.NodeId()); - TimerStarted = true; - OnFailure(); - }) - ) - - void Handle(typename EventType::TPtr ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - Callback(ev); - this->PassAway(); - } - + + TRichActorFutureCallback(TCallback&& callback, TFailure&& failure, TDuration timeout) + : TBase(&TRichActorFutureCallback::StateWaitForEvent) + , Callback(std::move(callback)) + , Failure(std::move(failure)) + , Timeout(timeout) + { } + +private: + const TCallback Callback; + const TFailure Failure; + const TDuration Timeout; + bool TimerStarted = false; + NActors::TSchedulerCookieHolder TimerCookieHolder; + + STRICT_STFUNC(StateWaitForEvent, + HFunc(EventType, Handle) + cFunc(NActors::TEvents::TEvBootstrap::EventType, OnFailure) + hFunc(NActors::TEvInterconnect::TEvNodeConnected, [this] (NActors::TEvInterconnect::TEvNodeConnected::TPtr& ev) mutable { + this->Subscribe(ev->Get()->NodeId); + }) + hFunc(NActors::TEvInterconnect::TEvNodeDisconnected, [this] (NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) mutable { + this->Unsubscribe(ev->Get()->NodeId); + TimerStarted = true; + OnFailure(); + }) + hFunc(NActors::TEvents::TEvUndelivered, [this] (NActors::TEvents::TEvUndelivered::TPtr& ev) mutable { + this->Unsubscribe(ev->Sender.NodeId()); + TimerStarted = true; + OnFailure(); + }) + ) + + void Handle(typename EventType::TPtr ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + Callback(ev); + this->PassAway(); + } + TAutoPtr<NActors::IEventHandle> AfterRegister(const NActors::TActorId& self, const NActors::TActorId& parentId) override { - return new NActors::IEventHandle(self, parentId, new NActors::TEvents::TEvBootstrap, 0); - } - - void OnFailure() { - if (TimerStarted) { - Failure(); - this->PassAway(); - } else { - TimerStarted = true; - TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); - this->Schedule(Timeout, new NActors::TEvents::TEvBootstrap, TimerCookieHolder.Get()); - } - } -}; - + return new NActors::IEventHandle(self, parentId, new NActors::TEvents::TEvBootstrap, 0); + } + + void OnFailure() { + if (TimerStarted) { + Failure(); + this->PassAway(); + } else { + TimerStarted = true; + TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); + this->Schedule(Timeout, new NActors::TEvents::TEvBootstrap, TimerCookieHolder.Get()); + } + } +}; + template <class TDerived> class TSynchronizableRichActor : public TRichActor<TDerived> { public: @@ -173,4 +173,4 @@ private: } }; -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/compute_actor.cpp b/ydb/library/yql/providers/dq/actors/compute_actor.cpp index 3b965ee60c..a2ae923d16 100644 --- a/ydb/library/yql/providers/dq/actors/compute_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/compute_actor.cpp @@ -1,86 +1,86 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> -#include <ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h> - +#include <ydb/library/yql/dq/actors/compute/dq_async_compute_actor.h> + #include <ydb/library/yql/providers/dq/api/protos/service.pb.h> #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h> - -#include <util/generic/size_literals.h> - -#include "compute_actor.h" - -namespace NYql { - -using namespace NActors; -using namespace NKikimr; -using namespace NDqs; - -IActor* CreateComputeActor( - const TLocalWorkerManagerOptions& options, - NDq::TAllocateMemoryCallback allocateMemoryFn, - NDq::TFreeMemoryCallback freeMemoryFn, - const TActorId& executerId, + +#include <util/generic/size_literals.h> + +#include "compute_actor.h" + +namespace NYql { + +using namespace NActors; +using namespace NKikimr; +using namespace NDqs; + +IActor* CreateComputeActor( + const TLocalWorkerManagerOptions& options, + NDq::TAllocateMemoryCallback allocateMemoryFn, + NDq::TFreeMemoryCallback freeMemoryFn, + const TActorId& executerId, const TString& operationId, - NYql::NDqProto::TDqTask&& task, - const TString& computeActorType, - const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) -{ + NYql::NDqProto::TDqTask&& task, + const TString& computeActorType, + const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory) +{ auto memoryLimits = NDq::TComputeMemoryLimits(); memoryLimits.ChannelBufferSize = 1000000; - memoryLimits.ScanBufferSize = 16_MB; + memoryLimits.ScanBufferSize = 16_MB; // light == heavy since we allow extra allocation - memoryLimits.MkqlLightProgramMemoryLimit = options.MkqlInitialMemoryLimit; - memoryLimits.MkqlHeavyProgramMemoryLimit = options.MkqlInitialMemoryLimit; + memoryLimits.MkqlLightProgramMemoryLimit = options.MkqlInitialMemoryLimit; + memoryLimits.MkqlHeavyProgramMemoryLimit = options.MkqlInitialMemoryLimit; memoryLimits.AllocateMemoryFn = allocateMemoryFn; memoryLimits.FreeMemoryFn = freeMemoryFn; // min alloc size == min free size to simplify api - memoryLimits.MinMemAllocSize = options.MkqlMinAllocSize; - memoryLimits.MinMemFreeSize = options.MkqlMinAllocSize; - + memoryLimits.MinMemAllocSize = options.MkqlMinAllocSize; + memoryLimits.MinMemFreeSize = options.MkqlMinAllocSize; + auto computeRuntimeSettings = NDq::TComputeRuntimeSettings(); computeRuntimeSettings.ExtraMemoryAllocationPool = 3; computeRuntimeSettings.FailOnUndelivery = false; computeRuntimeSettings.StatsMode = NDqProto::DQ_STATS_MODE_PROFILE; - - // clear fake actorids - for (auto& input : *task.MutableInputs()) { - for (auto& channel : *input.MutableChannels()) { - channel.MutableSrcEndpoint()->ClearActorId(); - channel.MutableDstEndpoint()->ClearActorId(); - } - } - for (auto& output : *task.MutableOutputs()) { - for (auto& channel : *output.MutableChannels()) { - channel.MutableSrcEndpoint()->ClearActorId(); - channel.MutableDstEndpoint()->ClearActorId(); - } - } - + + // clear fake actorids + for (auto& input : *task.MutableInputs()) { + for (auto& channel : *input.MutableChannels()) { + channel.MutableSrcEndpoint()->ClearActorId(); + channel.MutableDstEndpoint()->ClearActorId(); + } + } + for (auto& output : *task.MutableOutputs()) { + for (auto& channel : *output.MutableChannels()) { + channel.MutableSrcEndpoint()->ClearActorId(); + channel.MutableDstEndpoint()->ClearActorId(); + } + } + auto taskRunnerFactory = [=](const NDqProto::TDqTask& task, const NDq::TLogFunc& logger) { - Y_UNUSED(logger); - return options.Factory->Get(task, {}); - }; - - if (computeActorType.empty() || computeActorType == "old") { - return NYql::NDq::CreateDqComputeActor( - executerId, - operationId, - std::move(task), - std::move(options.SourceActorFactory), - std::move(options.SinkActorFactory), - computeRuntimeSettings, - memoryLimits, - taskRunnerFactory); - } else { - return NYql::NDq::CreateDqAsyncComputeActor( - executerId, - operationId, - std::move(task), - std::move(options.SourceActorFactory), - std::move(options.SinkActorFactory), - computeRuntimeSettings, - memoryLimits, - taskRunnerActorFactory); - } -} - -} /* namespace NYql */ + Y_UNUSED(logger); + return options.Factory->Get(task, {}); + }; + + if (computeActorType.empty() || computeActorType == "old") { + return NYql::NDq::CreateDqComputeActor( + executerId, + operationId, + std::move(task), + std::move(options.SourceActorFactory), + std::move(options.SinkActorFactory), + computeRuntimeSettings, + memoryLimits, + taskRunnerFactory); + } else { + return NYql::NDq::CreateDqAsyncComputeActor( + executerId, + operationId, + std::move(task), + std::move(options.SourceActorFactory), + std::move(options.SinkActorFactory), + computeRuntimeSettings, + memoryLimits, + taskRunnerActorFactory); + } +} + +} /* namespace NYql */ diff --git a/ydb/library/yql/providers/dq/actors/compute_actor.h b/ydb/library/yql/providers/dq/actors/compute_actor.h index 57ca847a46..2bf024dfc3 100644 --- a/ydb/library/yql/providers/dq/actors/compute_actor.h +++ b/ydb/library/yql/providers/dq/actors/compute_actor.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - -namespace NYql { - -NActors::IActor* CreateComputeActor( - const NYql::NDqs::TLocalWorkerManagerOptions& options, - NDq::TAllocateMemoryCallback allocateMemoryFn, - NDq::TFreeMemoryCallback freeMemoryFn, - const NActors::TActorId& executerId, - const TString& operationId, - NYql::NDqProto::TDqTask&& task, - const TString& computeActorType, - const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory); - -} // namespace NYql + +namespace NYql { + +NActors::IActor* CreateComputeActor( + const NYql::NDqs::TLocalWorkerManagerOptions& options, + NDq::TAllocateMemoryCallback allocateMemoryFn, + NDq::TFreeMemoryCallback freeMemoryFn, + const NActors::TActorId& executerId, + const TString& operationId, + NYql::NDqProto::TDqTask&& task, + const TString& computeActorType, + const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/events.cpp b/ydb/library/yql/providers/dq/actors/events.cpp index 69737860b9..892c5cccd7 100644 --- a/ydb/library/yql/providers/dq/actors/events.cpp +++ b/ydb/library/yql/providers/dq/actors/events.cpp @@ -3,30 +3,30 @@ #include <ydb/library/yql/public/issue/yql_issue_message.h> namespace NYql::NDqs { - TEvDqTask::TEvDqTask(NDqProto::TDqTask task) { - *Record.MutableTask() = std::move(task); - } - + TEvDqTask::TEvDqTask(NDqProto::TDqTask task) { + *Record.MutableTask() = std::move(task); + } + TEvDqFailure::TEvDqFailure(const TIssue& issue, bool retriable, bool needFallback) { IssuesToMessage({issue}, Record.MutableIssues()); - Record.SetRetriable(retriable); - Record.SetNeedFallback(needFallback); - } - + Record.SetRetriable(retriable); + Record.SetNeedFallback(needFallback); + } + TEvQueryResponse::TEvQueryResponse(NDqProto::TQueryResponse&& queryResult) { Record = std::move(queryResult); } TEvGraphRequest::TEvGraphRequest(const Yql::DqsProto::ExecuteGraphRequest& request, NActors::TActorId controlId, NActors::TActorId resultId, NActors::TActorId checkPointCoordinatorId) - { - *Record.MutableRequest() = request; + { + *Record.MutableRequest() = request; NActors::ActorIdToProto(controlId, Record.MutableControlId()); NActors::ActorIdToProto(resultId, Record.MutableResultId()); if (checkPointCoordinatorId) { NActors::ActorIdToProto(checkPointCoordinatorId, Record.MutableCheckPointCoordinatorId()); } - } - + } + TEvReadyState::TEvReadyState(NActors::TActorId sourceId, TString type) { NActors::ActorIdToProto(sourceId, Record.MutableSourceId()); *Record.MutableResultType() = std::move(type); diff --git a/ydb/library/yql/providers/dq/actors/events.h b/ydb/library/yql/providers/dq/actors/events.h index 6906f4c2b5..594921560a 100644 --- a/ydb/library/yql/providers/dq/actors/events.h +++ b/ydb/library/yql/providers/dq/actors/events.h @@ -14,27 +14,27 @@ namespace NYql::NDqs { using TDqExecuterEvents = NDq::TBaseDqExecuterEvents<NActors::TEvents::EEventSpace::ES_USERSPACE>; - struct TEvDqTask : NActors::TEventPB<TEvDqTask, NDqProto::TDqTaskRequest, TDqExecuterEvents::ES_DQ_TASK> { + struct TEvDqTask : NActors::TEventPB<TEvDqTask, NDqProto::TDqTaskRequest, TDqExecuterEvents::ES_DQ_TASK> { TEvDqTask() = default; explicit TEvDqTask(NDqProto::TDqTask task); }; - struct TEvDqFailure : NActors::TEventPB<TEvDqFailure, NDqProto::TDqFailure, TDqExecuterEvents::ES_DQ_FAILURE> { - TEvDqFailure() = default; + struct TEvDqFailure : NActors::TEventPB<TEvDqFailure, NDqProto::TDqFailure, TDqExecuterEvents::ES_DQ_FAILURE> { + TEvDqFailure() = default; explicit TEvDqFailure(const TIssue& issue, bool retriable = false, bool needFallback = false); - }; - + }; + struct TEvQueryResponse : NActors::TEventPB<TEvQueryResponse, NDqProto::TQueryResponse, TDqExecuterEvents::ES_RESULT_SET> { TEvQueryResponse() = default; explicit TEvQueryResponse(NDqProto::TQueryResponse&& queryResult); }; - struct TEvGraphRequest : NActors::TEventPB<TEvGraphRequest, NDqProto::TGraphRequest, TDqExecuterEvents::ES_GRAPH> { - TEvGraphRequest() = default; + struct TEvGraphRequest : NActors::TEventPB<TEvGraphRequest, NDqProto::TGraphRequest, TDqExecuterEvents::ES_GRAPH> { + TEvGraphRequest() = default; TEvGraphRequest(const Yql::DqsProto::ExecuteGraphRequest& request, NActors::TActorId controlId, NActors::TActorId resultId, NActors::TActorId checkPointCoordinatorId = {}); - }; - + }; + struct TEvReadyState : NActors::TEventPB<TEvReadyState, NDqProto::TReadyState, TDqExecuterEvents::ES_READY_TO_PULL> { TEvReadyState() = default; TEvReadyState(NActors::TActorId sourceId, TString type); @@ -64,16 +64,16 @@ namespace NYql::NDqs { TEvPullDataResponse() = default; explicit TEvPullDataResponse(NDqProto::TPullResponse& data); }; - - struct TEvPingRequest - : NActors::TEventPB<TEvPingRequest, NYql::NDqProto::TPingRequest, TDqDataEvents::ES_PING_REQUEST> { - TEvPingRequest() = default; - }; - - struct TEvPingResponse - : NActors::TEventPB<TEvPingResponse, NYql::NDqProto::TPingResponse, TDqDataEvents::ES_PING_RESPONSE> { - TEvPingResponse() = default; - }; + + struct TEvPingRequest + : NActors::TEventPB<TEvPingRequest, NYql::NDqProto::TPingRequest, TDqDataEvents::ES_PING_REQUEST> { + TEvPingRequest() = default; + }; + + struct TEvPingResponse + : NActors::TEventPB<TEvPingResponse, NYql::NDqProto::TPingResponse, TDqDataEvents::ES_PING_RESPONSE> { + TEvPingResponse() = default; + }; struct TEvFullResultWriterStatusRequest : NActors::TEventPB<TEvFullResultWriterStatusRequest, NYql::NDqProto::TFullResultWriterStatusRequest, diff --git a/ydb/library/yql/providers/dq/actors/executer_actor.cpp b/ydb/library/yql/providers/dq/actors/executer_actor.cpp index e984ad2993..91f1caee74 100644 --- a/ydb/library/yql/providers/dq/actors/executer_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/executer_actor.cpp @@ -1,11 +1,11 @@ #include "executer_actor.h" -#include "resource_allocator.h" +#include "resource_allocator.h" + +#include "execution_helpers.h" -#include "execution_helpers.h" - #include <ydb/library/yql/providers/dq/actors/events.h> #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> - + #include <ydb/library/yql/providers/dq/planner/execution_planner.h> #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> @@ -14,124 +14,124 @@ #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/protobuf/util/pb_io.h> +#include <library/cpp/protobuf/util/pb_io.h> #include <ydb/library/yql/utils/failure_injector/failure_injector.h> #include <ydb/library/yql/providers/dq/counters/counters.h> #include <ydb/library/yql/providers/dq/api/protos/service.pb.h> -#include <util/string/split.h> -#include <util/system/env.h> - -namespace NYql { -namespace NDq { - -using namespace NMonitoring; -using namespace NActors; -using namespace NKikimr::NMiniKQL; -using namespace NYql::NDqProto; -using namespace NYql::NDqs; -using namespace NYql; - -class TDqExecuter: public TRichActor<TDqExecuter>, NYql::TCounters { -public: +#include <util/string/split.h> +#include <util/system/env.h> + +namespace NYql { +namespace NDq { + +using namespace NMonitoring; +using namespace NActors; +using namespace NKikimr::NMiniKQL; +using namespace NYql::NDqProto; +using namespace NYql::NDqs; +using namespace NYql; + +class TDqExecuter: public TRichActor<TDqExecuter>, NYql::TCounters { +public: static constexpr char ActorName[] = "YQL_DQ_EXECUTER"; - - TDqExecuter( + + TDqExecuter( const NActors::TActorId& gwmActorId, const NActors::TActorId& printerId, const TString& traceId, const TString& username, - const TDqConfiguration::TPtr& settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TDqConfiguration::TPtr& settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, TInstant requestStartTime, bool createTaskSuspended) - : TRichActor<TDqExecuter>(&TDqExecuter::Handler) + : TRichActor<TDqExecuter>(&TDqExecuter::Handler) , GwmActorId(gwmActorId) , PrinterId(printerId) , Settings(settings) - , TraceId(traceId) + , TraceId(traceId) , Username(username) - , Counters(counters) // root, component=dq - , LongWorkersAllocationCounter(Counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("LongWorkersAllocation")) - , ExecutionTimeoutCounter(Counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("ExecutionTimeout", /*derivative=*/ true)) - , RequestStartTime(requestStartTime) - , ExecutionHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("ExecutionTime", ExponentialHistogram(10, 3, 1))) - , AllocationHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("WorkersAllocationTime", ExponentialHistogram(10, 2, 1))) - , TasksHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("TasksCount", ExponentialHistogram(10, 2, 1))) + , Counters(counters) // root, component=dq + , LongWorkersAllocationCounter(Counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("LongWorkersAllocation")) + , ExecutionTimeoutCounter(Counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("ExecutionTimeout", /*derivative=*/ true)) + , RequestStartTime(requestStartTime) + , ExecutionHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("ExecutionTime", ExponentialHistogram(10, 3, 1))) + , AllocationHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("WorkersAllocationTime", ExponentialHistogram(10, 2, 1))) + , TasksHistogram(Counters->GetSubgroup("component", "ServiceProxyActorHistograms")->GetHistogram("TasksCount", ExponentialHistogram(10, 2, 1))) , CreateTaskSuspended(createTaskSuspended) - { } - - ~TDqExecuter() { - MaybeResetAllocationWarnCounter(); - - if (ExecutionStart) { - ExecutionHistogram->Collect((TInstant::Now() - ExecutionStart).Seconds()); - } - } - -private: - - STRICT_STFUNC(Handler, { - HFunc(TEvGraphRequest, OnGraph); - HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse); - cFunc(TEvents::TEvPoison::EventType, PassAway); + { } + + ~TDqExecuter() { + MaybeResetAllocationWarnCounter(); + + if (ExecutionStart) { + ExecutionHistogram->Collect((TInstant::Now() - ExecutionStart).Seconds()); + } + } + +private: + + STRICT_STFUNC(Handler, { + HFunc(TEvGraphRequest, OnGraph); + HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse); + cFunc(TEvents::TEvPoison::EventType, PassAway); hFunc(NActors::TEvents::TEvPoisonTaken, Handle); HFunc(TEvDqFailure, OnFailure); HFunc(TEvGraphFinished, OnGraphFinished); HFunc(TEvQueryResponse, OnQueryResponse); - // execution timeout - cFunc(TEvents::TEvBootstrap::EventType, [this]() { + // execution timeout + cFunc(TEvents::TEvBootstrap::EventType, [this]() { YQL_LOG_CTX_SCOPE(TraceId); YQL_LOG(DEBUG) << "Execution timeout"; - auto issue = TIssue("Execution timeout"); - issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR); - Issues.AddIssues({issue}); - *ExecutionTimeoutCounter += 1; + auto issue = TIssue("Execution timeout"); + issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR); + Issues.AddIssues({issue}); + *ExecutionTimeoutCounter += 1; Finish(/*retriable=*/ false, /*needFallback=*/ true); - }) - cFunc(TEvents::TEvWakeup::EventType, OnWakeup) + }) + cFunc(TEvents::TEvWakeup::EventType, OnWakeup) }) - Yql::DqsProto::TWorkerFilter GetPragmaFilter() { - Yql::DqsProto::TWorkerFilter pragmaFilter; - if (Settings->WorkerFilter.Get()) { - try { - TStringInput inputStream1(Settings->WorkerFilter.Get().GetOrElse("")); - ParseFromTextFormat(inputStream1, pragmaFilter); - } catch (...) { - YQL_LOG(INFO) << "Cannot parse filter pragma " << CurrentExceptionMessage(); - } - } - return pragmaFilter; - } - - void MergeFilter(Yql::DqsProto::TWorkerFilter* to, const Yql::DqsProto::TWorkerFilter& from) - { -#define COPY(name) \ - if (from. Get##name()) { \ - *to->Mutable##name() = from.Get##name(); \ - } -#define COPYAR(name) \ - if (from. Get##name().size()>0) { \ - *to->Mutable##name() = from.Get##name(); \ - } - - COPY(Revision); - COPY(ClusterName); - COPY(ClusterNameHint); - COPYAR(Address); - COPYAR(NodeId); - COPYAR(NodeIdHint); - -#undef COPY -#undef COPYARR - } - - void OnGraph(TEvGraphRequest::TPtr& ev, const NActors::TActorContext&) { - YQL_LOG_CTX_SCOPE(TraceId); + Yql::DqsProto::TWorkerFilter GetPragmaFilter() { + Yql::DqsProto::TWorkerFilter pragmaFilter; + if (Settings->WorkerFilter.Get()) { + try { + TStringInput inputStream1(Settings->WorkerFilter.Get().GetOrElse("")); + ParseFromTextFormat(inputStream1, pragmaFilter); + } catch (...) { + YQL_LOG(INFO) << "Cannot parse filter pragma " << CurrentExceptionMessage(); + } + } + return pragmaFilter; + } + + void MergeFilter(Yql::DqsProto::TWorkerFilter* to, const Yql::DqsProto::TWorkerFilter& from) + { +#define COPY(name) \ + if (from. Get##name()) { \ + *to->Mutable##name() = from.Get##name(); \ + } +#define COPYAR(name) \ + if (from. Get##name().size()>0) { \ + *to->Mutable##name() = from.Get##name(); \ + } + + COPY(Revision); + COPY(ClusterName); + COPY(ClusterNameHint); + COPYAR(Address); + COPYAR(NodeId); + COPYAR(NodeIdHint); + +#undef COPY +#undef COPYARR + } + + void OnGraph(TEvGraphRequest::TPtr& ev, const NActors::TActorContext&) { + YQL_LOG_CTX_SCOPE(TraceId); Y_VERIFY(!ControlId); - Y_VERIFY(!ResultId); - YQL_LOG(DEBUG) << "TDqExecuter::OnGraph"; + Y_VERIFY(!ResultId); + YQL_LOG(DEBUG) << "TDqExecuter::OnGraph"; ControlId = NActors::ActorIdFromProto(ev->Get()->Record.GetControlId()); ResultId = NActors::ActorIdFromProto(ev->Get()->Record.GetResultId()); CheckPointCoordinatorId = NActors::ActorIdFromProto(ev->Get()->Record.GetCheckPointCoordinatorId()); @@ -140,104 +140,104 @@ private: AddChild(ResultId); AddChild(CheckPointCoordinatorId); - int workerCount = ev->Get()->Record.GetRequest().GetTask().size(); - YQL_LOG(INFO) << (TStringBuilder() << "Trying to allocate " << workerCount << " workers"); - + int workerCount = ev->Get()->Record.GetRequest().GetTask().size(); + YQL_LOG(INFO) << (TStringBuilder() << "Trying to allocate " << workerCount << " workers"); + THashMap<TString, Yql::DqsProto::TFile> files; - TVector<NDqProto::TDqTask> tasks; - for (auto& task : *ev->Get()->Record.MutableRequest()->MutableTask()) { - Yql::DqsProto::TTaskMeta taskMeta; - task.GetMeta().UnpackTo(&taskMeta); - - for (const auto& f : taskMeta.GetFiles()) { + TVector<NDqProto::TDqTask> tasks; + for (auto& task : *ev->Get()->Record.MutableRequest()->MutableTask()) { + Yql::DqsProto::TTaskMeta taskMeta; + task.GetMeta().UnpackTo(&taskMeta); + + for (const auto& f : taskMeta.GetFiles()) { files.emplace(f.GetObjectId(), f); - } - - if (ev->Get()->Record.GetRequest().GetSecureParams().size() > 0) { - *taskMeta.MutableSecureParams() = ev->Get()->Record.GetRequest().GetSecureParams(); - } - - Settings->Save(taskMeta); - - task.MutableMeta()->PackFrom(taskMeta); + } + + if (ev->Get()->Record.GetRequest().GetSecureParams().size() > 0) { + *taskMeta.MutableSecureParams() = ev->Get()->Record.GetRequest().GetSecureParams(); + } + + Settings->Save(taskMeta); + + task.MutableMeta()->PackFrom(taskMeta); task.SetCreateSuspended(CreateTaskSuspended); - - tasks.emplace_back(task); - } - TasksHistogram->Collect(tasks.size()); - - ExecutionPlanner = THolder<IDqsExecutionPlanner>(new TGraphExecutionPlanner( - tasks, - ev->Get()->Record.GetRequest().GetSourceId(), - ev->Get()->Record.GetRequest().GetResultType(), - SelfId(), ResultId)); - - + + tasks.emplace_back(task); + } + TasksHistogram->Collect(tasks.size()); + + ExecutionPlanner = THolder<IDqsExecutionPlanner>(new TGraphExecutionPlanner( + tasks, + ev->Get()->Record.GetRequest().GetSourceId(), + ev->Get()->Record.GetRequest().GetResultType(), + SelfId(), ResultId)); + + const bool enableComputeActor = Settings->EnableComputeActor.Get().GetOrElse(false); - const TString computeActorType = Settings->ComputeActorType.Get().GetOrElse("old"); - - auto resourceAllocator = RegisterChild(CreateResourceAllocator( + const TString computeActorType = Settings->ComputeActorType.Get().GetOrElse("old"); + + auto resourceAllocator = RegisterChild(CreateResourceAllocator( GwmActorId, SelfId(), ControlId, workerCount, - TraceId, Settings, - Counters, - enableComputeActor ? tasks : TVector<NYql::NDqProto::TDqTask>(), - computeActorType)); + TraceId, Settings, + Counters, + enableComputeActor ? tasks : TVector<NYql::NDqProto::TDqTask>(), + computeActorType)); auto allocateRequest = MakeHolder<TEvAllocateWorkersRequest>(workerCount, Username); - allocateRequest->Record.SetTraceId(TraceId); + allocateRequest->Record.SetTraceId(TraceId); allocateRequest->Record.SetCreateComputeActor(enableComputeActor); - allocateRequest->Record.SetComputeActorType(computeActorType); - if (enableComputeActor) { + allocateRequest->Record.SetComputeActorType(computeActorType); + if (enableComputeActor) { ActorIdToProto(ControlId, allocateRequest->Record.MutableResultActorId()); - } + } for (const auto& [_, f] : files) { - *allocateRequest->Record.AddFiles() = f; - } - - if (Settings->WorkersPerOperation.Get()) { - allocateRequest->Record.SetWorkersCount(*Settings->WorkersPerOperation.Get()); - } - - Yql::DqsProto::TWorkerFilter pragmaFilter = GetPragmaFilter(); - + *allocateRequest->Record.AddFiles() = f; + } + + if (Settings->WorkersPerOperation.Get()) { + allocateRequest->Record.SetWorkersCount(*Settings->WorkersPerOperation.Get()); + } + + Yql::DqsProto::TWorkerFilter pragmaFilter = GetPragmaFilter(); + for (const auto& task : tasks) { - Yql::DqsProto::TTaskMeta taskMeta; - task.GetMeta().UnpackTo(&taskMeta); - - Yql::DqsProto::TWorkerFilter* filter = allocateRequest->Record.AddWorkerFilterPerTask(); - *filter->MutableFile() = taskMeta.GetFiles(); - filter->SetClusterNameHint(taskMeta.GetClusterNameHint()); - - if (enableComputeActor) { - *allocateRequest->Record.AddTask() = task; - } - - MergeFilter(filter, pragmaFilter); + Yql::DqsProto::TTaskMeta taskMeta; + task.GetMeta().UnpackTo(&taskMeta); + + Yql::DqsProto::TWorkerFilter* filter = allocateRequest->Record.AddWorkerFilterPerTask(); + *filter->MutableFile() = taskMeta.GetFiles(); + filter->SetClusterNameHint(taskMeta.GetClusterNameHint()); + + if (enableComputeActor) { + *allocateRequest->Record.AddTask() = task; + } + + MergeFilter(filter, pragmaFilter); } - StartCounter("AllocateWorkers"); - - TActivationContext::Send(new IEventHandle( + StartCounter("AllocateWorkers"); + + TActivationContext::Send(new IEventHandle( GwmActorId, - resourceAllocator, - allocateRequest.Release(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession)); - - Timeout = tasks.size() == 1 - ? TDuration::MilliSeconds(Settings->_LiteralTimeout.Get().GetOrElse(TDqSettings::TDefault::LiteralTimeout)) - : TDuration::MilliSeconds(Settings->_TableTimeout.Get().GetOrElse(TDqSettings::TDefault::TableTimeout)); - - - if (Timeout) { - if (StartTime - RequestStartTime > Timeout) { - Send(SelfId(), new TEvents::TEvBootstrap()); - } else { - Timeout -= StartTime - RequestStartTime; - } - } - } - + resourceAllocator, + allocateRequest.Release(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession)); + + Timeout = tasks.size() == 1 + ? TDuration::MilliSeconds(Settings->_LiteralTimeout.Get().GetOrElse(TDqSettings::TDefault::LiteralTimeout)) + : TDuration::MilliSeconds(Settings->_TableTimeout.Get().GetOrElse(TDqSettings::TDefault::TableTimeout)); + + + if (Timeout) { + if (StartTime - RequestStartTime > Timeout) { + Send(SelfId(), new TEvents::TEvBootstrap()); + } else { + Timeout -= StartTime - RequestStartTime; + } + } + } + void Finish(bool retriable, bool needFallback = false) - { + { YQL_LOG(DEBUG) << __FUNCTION__ << ", retriable=" << retriable << ", needFallback=" << needFallback; if (Finished) { YQL_LOG(WARN) << "Re-Finish IGNORED with Retriable=" << retriable << ", NeedFallback=" << needFallback; @@ -251,9 +251,9 @@ private: Send(ControlId, MakeHolder<TEvQueryResponse>(std::move(result))); Finished = true; } - } - - void OnFailure(TEvDqFailure::TPtr& ev, const NActors::TActorContext&) { + } + + void OnFailure(TEvDqFailure::TPtr& ev, const NActors::TActorContext&) { if (!Finished) { YQL_LOG_CTX_SCOPE(TraceId); YQL_LOG(DEBUG) << __FUNCTION__; @@ -266,9 +266,9 @@ private: Issues.AddIssues(issues); } Finish(retriable, fallback); - } - } - + } + } + void OnGraphFinished(TEvGraphFinished::TPtr&, const NActors::TActorContext&) { YQL_LOG_CTX_SCOPE(TraceId); YQL_LOG(DEBUG) << __FUNCTION__; @@ -281,7 +281,7 @@ private: Issues.AddIssue(TIssue("FailureInjection")); Finish(true); } - } + } } // TBD: wait for PoisonTaken from CheckPointCoordinator before send TEvQueryResponse to PrinterId @@ -297,94 +297,94 @@ private: // ignore ack from checkpoint coordinator now } - void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const NActors::TActorContext&) { - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "TDqExecuter::TEvAllocateWorkersResponse"; - - AddCounters(ev->Get()->Record); - FlushCounter("AllocateWorkers"); - + void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const NActors::TActorContext&) { + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "TDqExecuter::TEvAllocateWorkersResponse"; + + AddCounters(ev->Get()->Record); + FlushCounter("AllocateWorkers"); + auto& response = ev->Get()->Record; switch (response.GetTResponseCase()) { case TAllocateWorkersResponse::kWorkers: break; case TAllocateWorkersResponse::kError: { - YQL_LOG(ERROR) << "Error on allocate workers " - << ev->Get()->Record.GetError().GetMessage() << ":" - << static_cast<int>(ev->Get()->Record.GetError().GetErrorCode()); - Issues.AddIssue(TIssue(ev->Get()->Record.GetError().GetMessage()).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR)); + YQL_LOG(ERROR) << "Error on allocate workers " + << ev->Get()->Record.GetError().GetMessage() << ":" + << static_cast<int>(ev->Get()->Record.GetError().GetErrorCode()); + Issues.AddIssue(TIssue(ev->Get()->Record.GetError().GetMessage()).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR)); Finish(/*retriable = */ true, /*fallback = */ true); return; } - case TAllocateWorkersResponse::kNodes: + case TAllocateWorkersResponse::kNodes: case TAllocateWorkersResponse::TRESPONSE_NOT_SET: YQL_ENSURE(false, "Unexpected allocate result"); } auto& workerGroup = response.GetWorkers(); - ResourceId = workerGroup.GetResourceId(); - YQL_LOG(DEBUG) << "Allocated resource " << ResourceId; + ResourceId = workerGroup.GetResourceId(); + YQL_LOG(DEBUG) << "Allocated resource " << ResourceId; TVector<NActors::TActorId> workers; - for (const auto& actorIdProto : workerGroup.GetWorkerActor()) { + for (const auto& actorIdProto : workerGroup.GetWorkerActor()) { workers.emplace_back(NActors::ActorIdFromProto(actorIdProto)); } - - auto tasks = ExecutionPlanner->GetTasks(workers); - - THashMap<TString, Yql::DqsProto::TWorkerInfo> uniqueWorkers; - if (!workerGroup.GetWorker().empty()) { - ui32 i = 0; - for (const auto& workerInfo : workerGroup.GetWorker()) { - Yql::DqsProto::TTaskMeta taskMeta; - tasks[i].GetMeta().UnpackTo(&taskMeta); - - WorkerInfo[workerInfo.GetNodeId()] = std::make_tuple(workerInfo, taskMeta.GetStageId()); - - YQL_LOG(DEBUG) << "WorkerInfo: " << NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(workerInfo, taskMeta.GetStageId()); - YQL_LOG(DEBUG) << "TaskInfo: " << i << "/" << tasks[i].GetId(); - for (const auto& file: taskMeta.GetFiles()) { - YQL_LOG(DEBUG) << " ObjectId: " << file.GetObjectId(); - } - i++; - - uniqueWorkers.insert(std::make_pair(workerInfo.GetGuid(), workerInfo)); - } - AddCounter("UniqueWorkers", uniqueWorkers.size()); - } - - YQL_LOG(INFO) << workers.size() << " workers allocated"; - + + auto tasks = ExecutionPlanner->GetTasks(workers); + + THashMap<TString, Yql::DqsProto::TWorkerInfo> uniqueWorkers; + if (!workerGroup.GetWorker().empty()) { + ui32 i = 0; + for (const auto& workerInfo : workerGroup.GetWorker()) { + Yql::DqsProto::TTaskMeta taskMeta; + tasks[i].GetMeta().UnpackTo(&taskMeta); + + WorkerInfo[workerInfo.GetNodeId()] = std::make_tuple(workerInfo, taskMeta.GetStageId()); + + YQL_LOG(DEBUG) << "WorkerInfo: " << NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(workerInfo, taskMeta.GetStageId()); + YQL_LOG(DEBUG) << "TaskInfo: " << i << "/" << tasks[i].GetId(); + for (const auto& file: taskMeta.GetFiles()) { + YQL_LOG(DEBUG) << " ObjectId: " << file.GetObjectId(); + } + i++; + + uniqueWorkers.insert(std::make_pair(workerInfo.GetGuid(), workerInfo)); + } + AddCounter("UniqueWorkers", uniqueWorkers.size()); + } + + YQL_LOG(INFO) << workers.size() << " workers allocated"; + YQL_ENSURE(workers.size() == tasks.size()); - auto res = MakeHolder<TEvReadyState>(ExecutionPlanner->GetSourceID(), ExecutionPlanner->GetResultType()); - - if (Settings->EnableComputeActor.Get().GetOrElse(false) == false) { - for (size_t i = 0; i < tasks.size(); i++) { - // { fill debug info - Yql::DqsProto::TTaskMeta taskMeta; - tasks[i].GetMeta().UnpackTo(&taskMeta); - for (const auto& [_, v] : uniqueWorkers) { - *taskMeta.AddWorkerInfo() = v; - } - tasks[i].MutableMeta()->PackFrom(taskMeta); - // } - auto workerEv = MakeHolder<TEvDqTask>(std::move(tasks[i])); - Send(workers[i], workerEv.Release()); + auto res = MakeHolder<TEvReadyState>(ExecutionPlanner->GetSourceID(), ExecutionPlanner->GetResultType()); + + if (Settings->EnableComputeActor.Get().GetOrElse(false) == false) { + for (size_t i = 0; i < tasks.size(); i++) { + // { fill debug info + Yql::DqsProto::TTaskMeta taskMeta; + tasks[i].GetMeta().UnpackTo(&taskMeta); + for (const auto& [_, v] : uniqueWorkers) { + *taskMeta.AddWorkerInfo() = v; + } + tasks[i].MutableMeta()->PackFrom(taskMeta); + // } + auto workerEv = MakeHolder<TEvDqTask>(std::move(tasks[i])); + Send(workers[i], workerEv.Release()); + } + } else { + for (size_t i = 0; i < tasks.size(); i++) { + *res->Record.AddTask() = tasks[i]; + ActorIdToProto(workers[i], res->Record.AddActorId()); } - } else { - for (size_t i = 0; i < tasks.size(); i++) { - *res->Record.AddTask() = tasks[i]; - ActorIdToProto(workers[i], res->Record.AddActorId()); - } } - WorkersAllocated = true; - - ExecutionStart = TInstant::Now(); - StartCounter("ExecutionTime"); - - AllocationHistogram->Collect((ExecutionStart-StartTime).Seconds()); - + WorkersAllocated = true; + + ExecutionStart = TInstant::Now(); + StartCounter("ExecutionTime"); + + AllocationHistogram->Collect((ExecutionStart-StartTime).Seconds()); + auto readyState1 = res->Record; auto readyState2 = res->Record; Send(ControlId, res.Release()); @@ -394,99 +394,99 @@ private: if (CheckPointCoordinatorId) { Send(CheckPointCoordinatorId, new TEvReadyState(std::move(readyState2))); } - - if (Timeout) { - ExecutionTimeoutCookieHolder.Reset(ISchedulerCookie::Make2Way()); - Schedule(Timeout, new TEvents::TEvBootstrap, ExecutionTimeoutCookieHolder.Get()); - } + + if (Timeout) { + ExecutionTimeoutCookieHolder.Reset(ISchedulerCookie::Make2Way()); + Schedule(Timeout, new TEvents::TEvBootstrap, ExecutionTimeoutCookieHolder.Get()); + } + } + + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { + return new IEventHandle(self, parentId, new TEvents::TEvWakeup(), 0); + } + + void OnWakeup() { + if (WorkersAllocated) { + MaybeResetAllocationWarnCounter(); + } else { + MaybeSetAllocationWarnCounter(); + MaybeFailOnWorkersAllocation(); + CheckStateCookieHolder.Reset(ISchedulerCookie::Make2Way()); + Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup, CheckStateCookieHolder.Get()); + } + } + + void MaybeResetAllocationWarnCounter() { + if (AllocationLongWait) { + *LongWorkersAllocationCounter -= AllocationLongWait; + AllocationLongWait = 0; + } } - - TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { - return new IEventHandle(self, parentId, new TEvents::TEvWakeup(), 0); - } - - void OnWakeup() { - if (WorkersAllocated) { - MaybeResetAllocationWarnCounter(); - } else { - MaybeSetAllocationWarnCounter(); - MaybeFailOnWorkersAllocation(); - CheckStateCookieHolder.Reset(ISchedulerCookie::Make2Way()); - Schedule(TDuration::Seconds(1), new TEvents::TEvWakeup, CheckStateCookieHolder.Get()); - } - } - - void MaybeResetAllocationWarnCounter() { - if (AllocationLongWait) { - *LongWorkersAllocationCounter -= AllocationLongWait; - AllocationLongWait = 0; - } - } - - void MaybeFailOnWorkersAllocation() { - if (TInstant::Now() - StartTime > - TDuration::MilliSeconds(Settings->_LongWorkersAllocationFailTimeout.Get().GetOrElse(TDuration::Seconds(600).MilliSeconds()))) - { - Send(SelfId(), new TEvents::TEvBootstrap); - } - } - - void MaybeSetAllocationWarnCounter() { - if (TInstant::Now() - StartTime > - TDuration::MilliSeconds(Settings->_LongWorkersAllocationWarnTimeout.Get().GetOrElse(TDuration::Seconds(30).MilliSeconds()))) - { - if (!AllocationLongWait) { - AllocationLongWait = 1; - *LongWorkersAllocationCounter += AllocationLongWait; - } - } - } - + + void MaybeFailOnWorkersAllocation() { + if (TInstant::Now() - StartTime > + TDuration::MilliSeconds(Settings->_LongWorkersAllocationFailTimeout.Get().GetOrElse(TDuration::Seconds(600).MilliSeconds()))) + { + Send(SelfId(), new TEvents::TEvBootstrap); + } + } + + void MaybeSetAllocationWarnCounter() { + if (TInstant::Now() - StartTime > + TDuration::MilliSeconds(Settings->_LongWorkersAllocationWarnTimeout.Get().GetOrElse(TDuration::Seconds(30).MilliSeconds()))) + { + if (!AllocationLongWait) { + AllocationLongWait = 1; + *LongWorkersAllocationCounter += AllocationLongWait; + } + } + } + NActors::TActorId GwmActorId; NActors::TActorId PrinterId; TDqConfiguration::TPtr Settings; - + NActors::TActorId ControlId; NActors::TActorId ResultId; NActors::TActorId CheckPointCoordinatorId; - TExprNode::TPtr ExprRoot; - THolder<IDqsExecutionPlanner> ExecutionPlanner; - ui64 ResourceId = 0; - const TString TraceId; + TExprNode::TPtr ExprRoot; + THolder<IDqsExecutionPlanner> ExecutionPlanner; + ui64 ResourceId = 0; + const TString TraceId; const TString Username; - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TDynamicCounters::TCounterPtr LongWorkersAllocationCounter; - TDynamicCounters::TCounterPtr ExecutionTimeoutCounter; - - THashMap<ui32, std::tuple<Yql::DqsProto::TWorkerInfo, ui64>> WorkerInfo; // DEBUG - TDuration Timeout; - TSchedulerCookieHolder ExecutionTimeoutCookieHolder; - TSchedulerCookieHolder CheckStateCookieHolder; - bool WorkersAllocated = false; - const TInstant StartTime = TInstant::Now(); - const TInstant RequestStartTime; - int AllocationLongWait = 0; - TInstant ExecutionStart; - THistogramPtr ExecutionHistogram; - THistogramPtr AllocationHistogram; - THistogramPtr TasksHistogram; - - TIssues Issues; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + TDynamicCounters::TCounterPtr LongWorkersAllocationCounter; + TDynamicCounters::TCounterPtr ExecutionTimeoutCounter; + + THashMap<ui32, std::tuple<Yql::DqsProto::TWorkerInfo, ui64>> WorkerInfo; // DEBUG + TDuration Timeout; + TSchedulerCookieHolder ExecutionTimeoutCookieHolder; + TSchedulerCookieHolder CheckStateCookieHolder; + bool WorkersAllocated = false; + const TInstant StartTime = TInstant::Now(); + const TInstant RequestStartTime; + int AllocationLongWait = 0; + TInstant ExecutionStart; + THistogramPtr ExecutionHistogram; + THistogramPtr AllocationHistogram; + THistogramPtr TasksHistogram; + + TIssues Issues; bool CreateTaskSuspended; bool Finished = false; -}; - -NActors::IActor* MakeDqExecuter( +}; + +NActors::IActor* MakeDqExecuter( const NActors::TActorId& gwmActorId, const NActors::TActorId& printerId, const TString& traceId, const TString& username, - const TDqConfiguration::TPtr& settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TDqConfiguration::TPtr& settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, TInstant requestStartTime, bool createTaskSuspended -) { +) { return new TLogWrapReceive(new TDqExecuter(gwmActorId, printerId, traceId, username, settings, counters, requestStartTime, createTaskSuspended), traceId); -} - -} // namespace NDq -} // namespace NYql +} + +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/executer_actor.h b/ydb/library/yql/providers/dq/actors/executer_actor.h index 11172a6b00..fae7d1cdbf 100644 --- a/ydb/library/yql/providers/dq/actors/executer_actor.h +++ b/ydb/library/yql/providers/dq/actors/executer_actor.h @@ -3,20 +3,20 @@ #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <library/cpp/actors/core/actor.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> +#include <library/cpp/monlib/dynamic_counters/counters.h> -namespace NYql { -namespace NDq { - -NActors::IActor* MakeDqExecuter( +namespace NYql { +namespace NDq { + +NActors::IActor* MakeDqExecuter( const NActors::TActorId& gwmActorId, const NActors::TActorId& printerId, const TString& traceId, const TString& username, - const TDqConfiguration::TPtr& settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TDqConfiguration::TPtr& settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, TInstant requestStartTime = TInstant::Now(), bool createTaskSuspended = false -); - -} // namespace NDq -} // namespace NYql +); + +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/execution_helpers.cpp b/ydb/library/yql/providers/dq/actors/execution_helpers.cpp index e9f6f6d769..8288e493ef 100644 --- a/ydb/library/yql/providers/dq/actors/execution_helpers.cpp +++ b/ydb/library/yql/providers/dq/actors/execution_helpers.cpp @@ -4,33 +4,33 @@ #include <util/string/split.h> #include <util/system/types.h> - - -namespace NYql { - namespace NDqs::NExecutionHelpers { - - TString PrettyPrintWorkerInfo(const Yql::DqsProto::TWorkerInfo& workerInfo, ui64 stageId) { - TString result; - - { - TStringOutput output(result); - output << "Id:" << workerInfo.GetGuid() << ","; - output << "NodeId:" << workerInfo.GetNodeId() << ","; - if (!workerInfo.GetClusterName().empty()) { - output << "ClusterName:" << workerInfo.GetClusterName() << ","; - } - if (!workerInfo.GetJobId().empty()) { - output << "JobId:" << workerInfo.GetJobId() << ","; - } - if (!workerInfo.GetOperationId().empty()) { - output << "OperationId:" << workerInfo.GetOperationId() << ","; - } - if (stageId > 0) { - output << "StageId:" << stageId; - } - } - - return result; - } - } // namespace NDqs::NExecutionHelpers -} // namespace NYql + + +namespace NYql { + namespace NDqs::NExecutionHelpers { + + TString PrettyPrintWorkerInfo(const Yql::DqsProto::TWorkerInfo& workerInfo, ui64 stageId) { + TString result; + + { + TStringOutput output(result); + output << "Id:" << workerInfo.GetGuid() << ","; + output << "NodeId:" << workerInfo.GetNodeId() << ","; + if (!workerInfo.GetClusterName().empty()) { + output << "ClusterName:" << workerInfo.GetClusterName() << ","; + } + if (!workerInfo.GetJobId().empty()) { + output << "JobId:" << workerInfo.GetJobId() << ","; + } + if (!workerInfo.GetOperationId().empty()) { + output << "OperationId:" << workerInfo.GetOperationId() << ","; + } + if (stageId > 0) { + output << "StageId:" << stageId; + } + } + + return result; + } + } // namespace NDqs::NExecutionHelpers +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/execution_helpers.h b/ydb/library/yql/providers/dq/actors/execution_helpers.h index e9af839110..b2da06f2f1 100644 --- a/ydb/library/yql/providers/dq/actors/execution_helpers.h +++ b/ydb/library/yql/providers/dq/actors/execution_helpers.h @@ -2,27 +2,27 @@ #include <ydb/library/yql/providers/dq/api/protos/dqs.pb.h> -#include <util/generic/guid.h> +#include <util/generic/guid.h> namespace NYql::NDqs::NExecutionHelpers { - TString PrettyPrintWorkerInfo(const Yql::DqsProto::TWorkerInfo& workerInfo, ui64 stageId); - - template<typename T> - TGUID GuidFromProto(const T& t) { - TGUID guid; - guid.dw[0] = t.GetDw0(); - guid.dw[1] = t.GetDw1(); - guid.dw[2] = t.GetDw2(); - guid.dw[3] = t.GetDw3(); - return guid; - } - - template<typename T> - void GuidToProto(T& t, const TGUID& g) { - t.SetDw0(g.dw[0]); - t.SetDw1(g.dw[1]); - t.SetDw2(g.dw[2]); - t.SetDw3(g.dw[3]); - } - -} // namespace NYql::NDqs::NExecutionHelpers + TString PrettyPrintWorkerInfo(const Yql::DqsProto::TWorkerInfo& workerInfo, ui64 stageId); + + template<typename T> + TGUID GuidFromProto(const T& t) { + TGUID guid; + guid.dw[0] = t.GetDw0(); + guid.dw[1] = t.GetDw1(); + guid.dw[2] = t.GetDw2(); + guid.dw[3] = t.GetDw3(); + return guid; + } + + template<typename T> + void GuidToProto(T& t, const TGUID& g) { + t.SetDw0(g.dw[0]); + t.SetDw1(g.dw[1]); + t.SetDw2(g.dw[2]); + t.SetDw3(g.dw[3]); + } + +} // namespace NYql::NDqs::NExecutionHelpers diff --git a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp index 1635a1fcfa..927ac06043 100644 --- a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp +++ b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp @@ -1,6 +1,6 @@ #include "proto_builder.h" #include "full_result_writer.h" - + #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> #include <ydb/library/yql/providers/dq/actors/events.h> #include <ydb/library/yql/providers/dq/api/protos/service.pb.h> @@ -10,22 +10,22 @@ #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/failure_injector/failure_injector.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <library/cpp/actors/core/actor.h> -#include <util/generic/size_literals.h> +#include <util/generic/size_literals.h> #include <util/system/env.h> - -#include <utility> - -namespace NYql::NDqs { - -using namespace NKikimr::NMiniKQL; - + +#include <utility> + +namespace NYql::NDqs { + +using namespace NKikimr::NMiniKQL; + class TFullResultWriterActor : public NActors::TActor<TFullResultWriterActor> { -public: +public: static constexpr char ActorName[] = "YQL_DQ_FULL_RESULT_WRITER"; - + explicit TFullResultWriterActor(const TString& traceId, const TString& resultType, THolder<IDqFullResultWriter>&& writer, @@ -158,4 +158,4 @@ THolder<NActors::IActor> MakeFullResultWriterActor( return MakeHolder<TFullResultWriterActor>(traceId, resultType, std::move(writer), aggregatorId); } -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/actors/full_result_writer.h b/ydb/library/yql/providers/dq/actors/full_result_writer.h index b4cac9d771..4d4345522a 100644 --- a/ydb/library/yql/providers/dq/actors/full_result_writer.h +++ b/ydb/library/yql/providers/dq/actors/full_result_writer.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h> #include <library/cpp/actors/core/actor.h> -namespace NYql::NDqs { - +namespace NYql::NDqs { + THolder<NActors::IActor> MakeFullResultWriterActor( const TString& traceId, const TString& resultType, THolder<IDqFullResultWriter>&& writer, const NActors::TActorId& aggregatorId); -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp index 8278330e3d..3ff4fb1658 100644 --- a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp @@ -37,11 +37,11 @@ private: YQL_LOG_CTX_SCOPE(TraceID); YQL_LOG(DEBUG) << __FUNCTION__ << ' ' << NYql::NDqProto::EGraphExecutionEventType_Name(ev->Get()->Record.GetEventType()); - try { - switch (ev->Get()->Record.GetEventType()) { + try { + switch (ev->Get()->Record.GetEventType()) { case NYql::NDqProto::EGraphExecutionEventType::START: { - NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor payload; - ev->Get()->Record.GetMessage().UnpackTo(&payload); + NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor payload; + ev->Get()->Record.GetMessage().UnpackTo(&payload); OnStart(ev->Sender, payload); break; } @@ -60,10 +60,10 @@ private: default: Reply(ev->Sender); break; - } - } catch (...) { - TString message = TStringBuilder() << "Error on TEvGraphExecutionEvent: " << CurrentExceptionMessage(); - YQL_LOG(ERROR) << message; + } + } catch (...) { + TString message = TStringBuilder() << "Error on TEvGraphExecutionEvent: " << CurrentExceptionMessage(); + YQL_LOG(ERROR) << message; Reply(ev->Sender, message); } } @@ -94,9 +94,9 @@ private: void Reply(NActors::TActorId replyTo, TString msg = "") const { NYql::NDqProto::TGraphExecutionEvent sync; sync.SetEventType(NDqProto::EGraphExecutionEventType::SYNC); - if (msg) { - sync.SetErrorMessage(msg); - } + if (msg) { + sync.SetErrorMessage(msg); + } Send(replyTo, MakeHolder<NDqs::TEvGraphExecutionEvent>(sync)); } diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.cpp b/ydb/library/yql/providers/dq/actors/proto_builder.cpp index 50677085e6..e4b16403c3 100644 --- a/ydb/library/yql/providers/dq/actors/proto_builder.cpp +++ b/ydb/library/yql/providers/dq/actors/proto_builder.cpp @@ -11,19 +11,19 @@ #include <library/cpp/yson/node/node_io.h> #include <library/cpp/yson/writer.h> - + namespace NYql::NDqs { using namespace NKikimr::NMiniKQL; namespace { - + TVector<ui32> BuildColumnOrder(const TVector<TString>& columns, NKikimr::NMiniKQL::TType* resultType) { MKQL_ENSURE(resultType, "Incorrect result type"); if (resultType->GetKind() != TType::EKind::Struct || columns.empty()) { return {}; - } - + } + TVector<ui32> columnOrder; THashMap<TString, ui32> column2id; auto structType = AS_TYPE(TStructType, resultType); @@ -32,14 +32,14 @@ TVector<ui32> BuildColumnOrder(const TVector<TString>& columns, NKikimr::NMiniKQ column2id[columnName] = i; } columnOrder.resize(columns.size()); - + int id = 0; for (const auto& columnName : columns) { columnOrder[id++] = column2id[columnName]; } return columnOrder; } - + NDqProto::EDataTransportVersion GetTransportVersion(const NYql::NDqProto::TData& data) { switch (data.GetTransportVersion()) { case 10000: @@ -55,7 +55,7 @@ NDqProto::EDataTransportVersion GetTransportVersion(const NYql::NDqProto::TData& } } // unnamed - + TProtoBuilder::TProtoBuilder(const TString& type, const TVector<TString>& columns) : Alloc() , TypeEnv(Alloc) @@ -64,21 +64,21 @@ TProtoBuilder::TProtoBuilder(const TString& type, const TVector<TString>& column { Alloc.Release(); } - + TProtoBuilder::~TProtoBuilder() { Alloc.Acquire(); } - + bool TProtoBuilder::CanBuildResultSet() const { return ResultType->GetKind() == TType::EKind::Struct; } - + TString TProtoBuilder::BuildYson(const TVector<NYql::NDqProto::TData>& rows, ui64 maxBytesLimit) { ui64 size = 0; TStringStream out; NYson::TYsonWriter writer((IOutputStream*)&out); writer.OnBeginList(); - + auto full = WriteData(rows, [&](const NYql::NUdf::TUnboxedValuePod& value) { auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder); writer.OnListItem(); @@ -86,25 +86,25 @@ TString TProtoBuilder::BuildYson(const TVector<NYql::NDqProto::TData>& rows, ui6 size += rowYson.size(); return size <= maxBytesLimit; }); - + if (!full) { ythrow yexception() << "Too big yson result size: " << size << " > " << maxBytesLimit; - } - + } + writer.OnEndList(); return out.Str(); } - + bool TProtoBuilder::WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func) { return WriteData(data, [&](const NYql::NUdf::TUnboxedValuePod& value) { auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder); return func(rowYson); }); } - + bool TProtoBuilder::WriteData(const NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) { TGuard<TScopedAlloc> allocGuard(Alloc); - + TMemoryUsageInfo memInfo("ProtoBuilder"); THolderFactory holderFactory(Alloc.Ref(), memInfo); NDqProto::EDataTransportVersion transportVersion = GetTransportVersion(data); @@ -116,11 +116,11 @@ bool TProtoBuilder::WriteData(const NDqProto::TData& data, const std::function<b for (const auto& item : buffer) { if (!func(item)) { return false; - } - } + } + } return true; } - + bool TProtoBuilder::WriteData(const TVector<NDqProto::TData>& rows, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) { TGuard<TScopedAlloc> allocGuard(Alloc); diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.h b/ydb/library/yql/providers/dq/actors/proto_builder.h index 42e5d7bdd5..8ba05c8c3d 100644 --- a/ydb/library/yql/providers/dq/actors/proto_builder.h +++ b/ydb/library/yql/providers/dq/actors/proto_builder.h @@ -14,12 +14,12 @@ namespace NYql::NDqProto { } namespace NYql::NDqs { - + class TProtoBuilder { -public: +public: TProtoBuilder(const TString& type, const TVector<TString>& columns); ~TProtoBuilder(); - + bool CanBuildResultSet() const; Ydb::ResultSet BuildResultSet(const TVector<NYql::NDqProto::TData>& data); TString BuildYson(const TVector<NYql::NDqProto::TData>& data, ui64 maxBytesLimit = std::numeric_limits<ui64>::max()); @@ -37,4 +37,4 @@ private: }; } // NYql::NDqs - + diff --git a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp index 2262d622a6..6acfb1277e 100644 --- a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp +++ b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp @@ -1,343 +1,343 @@ -#include "resource_allocator.h" +#include "resource_allocator.h" #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> -#include "execution_helpers.h" - +#include "execution_helpers.h" + #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/library/yql/utils/log/log.h> - + #include <library/cpp/actors/core/hfunc.h> - + #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> #include <ydb/library/yql/providers/dq/counters/counters.h> - -namespace NYql { - -using namespace NActors; -using namespace NDqs; -using namespace NMonitoring; - -union TDqResourceId { - struct { - ui32 u1; - ui16 u2; - ui16 u3; - }; - ui64 Data; -}; - -class TResourceAllocator: public TRichActor<TResourceAllocator> -{ - struct TRequestInfo { - NYql::NDqProto::TDqTask Task; // for compute actor - Yql::DqsProto::TWorkerInfo WorkerInfo; - bool RequestedFlag; - TString ClusterName; - TInstant StartTime; - ui64 ResourceId; // for retries - int Retry = 0; - }; - - -public: + +namespace NYql { + +using namespace NActors; +using namespace NDqs; +using namespace NMonitoring; + +union TDqResourceId { + struct { + ui32 u1; + ui16 u2; + ui16 u3; + }; + ui64 Data; +}; + +class TResourceAllocator: public TRichActor<TResourceAllocator> +{ + struct TRequestInfo { + NYql::NDqProto::TDqTask Task; // for compute actor + Yql::DqsProto::TWorkerInfo WorkerInfo; + bool RequestedFlag; + TString ClusterName; + TInstant StartTime; + ui64 ResourceId; // for retries + int Retry = 0; + }; + + +public: static constexpr char ActorName[] = "YQL_DQ_RESOURCE_ALLOCATOR"; - - TResourceAllocator( + + TResourceAllocator( TActorId gwmActor, - TActorId senderId, + TActorId senderId, TActorId controlId, - ui32 workerCount, - const TString& traceId, - const TDqConfiguration::TPtr settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - const TVector<NYql::NDqProto::TDqTask>& tasks, - const TString& computeActorType) - : TRichActor<TResourceAllocator>(&TResourceAllocator::Handle) + ui32 workerCount, + const TString& traceId, + const TDqConfiguration::TPtr settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TVector<NYql::NDqProto::TDqTask>& tasks, + const TString& computeActorType) + : TRichActor<TResourceAllocator>(&TResourceAllocator::Handle) , GwmActor(gwmActor) - , SenderId(senderId) + , SenderId(senderId) , ControlId(controlId) - , RequestedCount(workerCount) - , TraceId(traceId) - , Settings(settings) - , NetworkRetries(Settings->MaxNetworkRetries.Get().GetOrElse(TDqSettings::TDefault::MaxNetworkRetries)) - , Timeout(GetAllocationTimeout(workerCount, Settings)) - , Counters(counters->GetSubgroup("counters", "Actor")) // root, component=dq, component=Actor - , RetryCounter(counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("RetryCreateActor", /*derivative=*/ true)) - , Tasks(tasks) - , ComputeActorType(computeActorType) + , RequestedCount(workerCount) + , TraceId(traceId) + , Settings(settings) + , NetworkRetries(Settings->MaxNetworkRetries.Get().GetOrElse(TDqSettings::TDefault::MaxNetworkRetries)) + , Timeout(GetAllocationTimeout(workerCount, Settings)) + , Counters(counters->GetSubgroup("counters", "Actor")) // root, component=dq, component=Actor + , RetryCounter(counters->GetSubgroup("component", "ServiceProxyActor")->GetCounter("RetryCreateActor", /*derivative=*/ true)) + , Tasks(tasks) + , ComputeActorType(computeActorType) { AllocatedWorkers.resize(workerCount); - if (!Tasks.empty()) { - Y_VERIFY(workerCount == Tasks.size()); - } + if (!Tasks.empty()) { + Y_VERIFY(workerCount == Tasks.size()); + } } - -private: - STRICT_STFUNC(Handle, { - HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse) - cFunc(TEvents::TEvPoison::EventType, PassAway) - hFunc(TEvInterconnect::TEvNodeConnected, [this](TEvInterconnect::TEvNodeConnected::TPtr& ev) { - // Store GWM NodeId. Auto-unsubscribe on actor-death - Subscribe(ev->Get()->NodeId); - }) - cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, [this]() { - Fail((ui64)-1, "GWM Disconnected"); - }) - hFunc(TEvents::TEvUndelivered, [this](TEvents::TEvUndelivered::TPtr& ev) { - Fail(ev->Cookie, "Undelivered"); - }) - }) - - static TDuration GetAllocationTimeout(int workerCount, const TDqConfiguration::TPtr settings) { - ui64 timeout = workerCount == 1 - ? settings->_LiteralTimeout.Get().GetOrElse(TDqSettings::TDefault::LiteralTimeout) - : settings->_TableTimeout.Get().GetOrElse(TDqSettings::TDefault::TableTimeout); - if (timeout & (1ULL<<63)) { - return TDuration(); // no timeout - } - return TDuration::MilliSeconds(timeout << 1); - } - - void RequestActorIdsFromNodes(TEvAllocateWorkersResponse::TPtr& ev, const TActorContext& ctx) - { - Y_UNUSED(ctx); - - YQL_LOG(DEBUG) << "RequestActorIdsFromNodes " << ev->Sender.NodeId(); - - auto& response = ev->Get()->Record; - auto& nodes = response.GetNodes().GetWorker(); - ResourceId = response.GetNodes().GetResourceId(); - Y_VERIFY(nodes.size() > 0); - Y_VERIFY(static_cast<ui32>(nodes.size()) == RequestedCount); - RequestedNodes.reserve(RequestedCount); - - YQL_LOG(DEBUG) << "RequestActorIdsFromNodes " << ev->Sender.NodeId() << " " << ResourceId; - auto now = TInstant::Now(); - ui16 i = 1; - for (const auto& node : nodes) { - YQL_LOG(DEBUG) << "RequestedNode: " << node.GetNodeId(); - TString clusterName = node.GetClusterName(); - NYql::NDqProto::TDqTask task; - if (!Tasks.empty()) { - task = Tasks[i-1]; - } - TDqResourceId resourceId; - resourceId.Data = ResourceId; - resourceId.u3 = i++; - RequestedNodes[resourceId.Data] = { task, node, false, clusterName, now, resourceId.Data }; - } - - for (const auto& [_, requestInfo] : RequestedNodes) { - SendToWorker(requestInfo); - } - } - - void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const TActorContext& ctx) - { - YQL_LOG_CTX_SCOPE(TraceId); - Y_UNUSED(ctx); - - YQL_LOG(DEBUG) << "TEvAllocateWorkersResponse " << ev->Sender.NodeId(); - - QueryStat.AddCounters(ev->Get()->Record); - - if (FailState) { - return; - } - - if (ev->Get()->Record.GetTResponseCase() == NDqProto::TAllocateWorkersResponse::kError) { - YQL_LOG(DEBUG) << "Forwarding bad state"; - ctx.Send(ev->Forward(SenderId)); - FailState = true; - return; - } - - if (ev->Get()->Record.GetTResponseCase() == NDqProto::TAllocateWorkersResponse::kNodes) { - Unsubscribe(ev->Sender.NodeId()); // don't need subscription anymore - return RequestActorIdsFromNodes(ev, ctx); - } - - if (RequestedNodes.empty()) { - /*work w/o gwm*/ - LocalMode = true; - AllocatedWorkers.emplace_back(ev->Get()->Record.GetWorkers()); - ctx.Send(ev->Forward(SenderId)); - return; - } - - auto cookie = ev->Cookie; - if (cookie == 0) { - // COMPAT - for (const auto& [_, v] : RequestedNodes) { - if (v.WorkerInfo.GetNodeId() == ev->Sender.NodeId()) { - cookie = v.ResourceId; - break; - } - } - } - YQL_ENSURE(RequestedNodes.contains(cookie)); - auto& requestedNode = RequestedNodes[cookie]; - if (!requestedNode.RequestedFlag) { + +private: + STRICT_STFUNC(Handle, { + HFunc(TEvAllocateWorkersResponse, OnAllocateWorkersResponse) + cFunc(TEvents::TEvPoison::EventType, PassAway) + hFunc(TEvInterconnect::TEvNodeConnected, [this](TEvInterconnect::TEvNodeConnected::TPtr& ev) { + // Store GWM NodeId. Auto-unsubscribe on actor-death + Subscribe(ev->Get()->NodeId); + }) + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, [this]() { + Fail((ui64)-1, "GWM Disconnected"); + }) + hFunc(TEvents::TEvUndelivered, [this](TEvents::TEvUndelivered::TPtr& ev) { + Fail(ev->Cookie, "Undelivered"); + }) + }) + + static TDuration GetAllocationTimeout(int workerCount, const TDqConfiguration::TPtr settings) { + ui64 timeout = workerCount == 1 + ? settings->_LiteralTimeout.Get().GetOrElse(TDqSettings::TDefault::LiteralTimeout) + : settings->_TableTimeout.Get().GetOrElse(TDqSettings::TDefault::TableTimeout); + if (timeout & (1ULL<<63)) { + return TDuration(); // no timeout + } + return TDuration::MilliSeconds(timeout << 1); + } + + void RequestActorIdsFromNodes(TEvAllocateWorkersResponse::TPtr& ev, const TActorContext& ctx) + { + Y_UNUSED(ctx); + + YQL_LOG(DEBUG) << "RequestActorIdsFromNodes " << ev->Sender.NodeId(); + + auto& response = ev->Get()->Record; + auto& nodes = response.GetNodes().GetWorker(); + ResourceId = response.GetNodes().GetResourceId(); + Y_VERIFY(nodes.size() > 0); + Y_VERIFY(static_cast<ui32>(nodes.size()) == RequestedCount); + RequestedNodes.reserve(RequestedCount); + + YQL_LOG(DEBUG) << "RequestActorIdsFromNodes " << ev->Sender.NodeId() << " " << ResourceId; + auto now = TInstant::Now(); + ui16 i = 1; + for (const auto& node : nodes) { + YQL_LOG(DEBUG) << "RequestedNode: " << node.GetNodeId(); + TString clusterName = node.GetClusterName(); + NYql::NDqProto::TDqTask task; + if (!Tasks.empty()) { + task = Tasks[i-1]; + } + TDqResourceId resourceId; + resourceId.Data = ResourceId; + resourceId.u3 = i++; + RequestedNodes[resourceId.Data] = { task, node, false, clusterName, now, resourceId.Data }; + } + + for (const auto& [_, requestInfo] : RequestedNodes) { + SendToWorker(requestInfo); + } + } + + void OnAllocateWorkersResponse(TEvAllocateWorkersResponse::TPtr& ev, const TActorContext& ctx) + { + YQL_LOG_CTX_SCOPE(TraceId); + Y_UNUSED(ctx); + + YQL_LOG(DEBUG) << "TEvAllocateWorkersResponse " << ev->Sender.NodeId(); + + QueryStat.AddCounters(ev->Get()->Record); + + if (FailState) { + return; + } + + if (ev->Get()->Record.GetTResponseCase() == NDqProto::TAllocateWorkersResponse::kError) { + YQL_LOG(DEBUG) << "Forwarding bad state"; + ctx.Send(ev->Forward(SenderId)); + FailState = true; + return; + } + + if (ev->Get()->Record.GetTResponseCase() == NDqProto::TAllocateWorkersResponse::kNodes) { + Unsubscribe(ev->Sender.NodeId()); // don't need subscription anymore + return RequestActorIdsFromNodes(ev, ctx); + } + + if (RequestedNodes.empty()) { + /*work w/o gwm*/ + LocalMode = true; + AllocatedWorkers.emplace_back(ev->Get()->Record.GetWorkers()); + ctx.Send(ev->Forward(SenderId)); + return; + } + + auto cookie = ev->Cookie; + if (cookie == 0) { + // COMPAT + for (const auto& [_, v] : RequestedNodes) { + if (v.WorkerInfo.GetNodeId() == ev->Sender.NodeId()) { + cookie = v.ResourceId; + break; + } + } + } + YQL_ENSURE(RequestedNodes.contains(cookie)); + auto& requestedNode = RequestedNodes[cookie]; + if (!requestedNode.RequestedFlag) { TDqResourceId resourceId{}; resourceId.Data = cookie; AllocatedWorkers[resourceId.u3 - 1] = ev->Get()->Record.GetWorkers(); AllocatedCount++; - requestedNode.RequestedFlag = true; - auto delta = TInstant::Now() - requestedNode.StartTime; - // catched and grpc_service - QueryStat.AddCounter(QueryStat.GetCounterName("Actor", {{"ClusterName", requestedNode.ClusterName}}, "ActorCreateTime"), delta); - } - + requestedNode.RequestedFlag = true; + auto delta = TInstant::Now() - requestedNode.StartTime; + // catched and grpc_service + QueryStat.AddCounter(QueryStat.GetCounterName("Actor", {{"ClusterName", requestedNode.ClusterName}}, "ActorCreateTime"), delta); + } + if (AllocatedCount == RequestedCount) { TVector<NActors::TActorId> workerIds; - for (auto& group : AllocatedWorkers) { - for (const auto& actorIdProto : group.GetWorkerActor()) { + for (auto& group : AllocatedWorkers) { + for (const auto& actorIdProto : group.GetWorkerActor()) { workerIds.emplace_back(NActors::ActorIdFromProto(actorIdProto)); - } - } - - auto response = MakeHolder<TEvAllocateWorkersResponse>(ResourceId, workerIds); - QueryStat.FlushCounters(response->Record); - auto* workerGroup = response->Record.MutableWorkers(); - TVector<Yql::DqsProto::TWorkerInfo> workers; + } + } + + auto response = MakeHolder<TEvAllocateWorkersResponse>(ResourceId, workerIds); + QueryStat.FlushCounters(response->Record); + auto* workerGroup = response->Record.MutableWorkers(); + TVector<Yql::DqsProto::TWorkerInfo> workers; workers.resize(AllocatedCount); for (const auto& [resourceId, requestInfo] : RequestedNodes) { TDqResourceId dqResourceId{}; dqResourceId.Data = resourceId; workers[dqResourceId.u3 - 1] = requestInfo.WorkerInfo; - } + } for (const auto& worker : workers) { *workerGroup->AddWorker() = worker; } - Send(SenderId, response.Release()); - Answered = true; - } - } - - void DoPassAway() override - { - for (const auto& group : AllocatedWorkers) { - for (const auto& actorIdProto : group.GetWorkerActor()) { + Send(SenderId, response.Release()); + Answered = true; + } + } + + void DoPassAway() override + { + for (const auto& group : AllocatedWorkers) { + for (const auto& actorIdProto : group.GetWorkerActor()) { auto actorNode = NActors::ActorIdFromProto(actorIdProto).NodeId(); - auto request = MakeHolder<TEvFreeWorkersNotify>(group.GetResourceId()); - request->Record.SetTraceId(TraceId); - Send(MakeWorkerManagerActorID(actorNode), request.Release()); - } - } - if (!LocalMode) { - auto request = MakeHolder<TEvFreeWorkersNotify>(ResourceId); - request->Record.SetTraceId(TraceId); - for (const auto& failedWorker : FailedWorkers) { - *request->Record.AddFailedWorkerGuid() = failedWorker.GetGuid(); - } + auto request = MakeHolder<TEvFreeWorkersNotify>(group.GetResourceId()); + request->Record.SetTraceId(TraceId); + Send(MakeWorkerManagerActorID(actorNode), request.Release()); + } + } + if (!LocalMode) { + auto request = MakeHolder<TEvFreeWorkersNotify>(ResourceId); + request->Record.SetTraceId(TraceId); + for (const auto& failedWorker : FailedWorkers) { + *request->Record.AddFailedWorkerGuid() = failedWorker.GetGuid(); + } Send(GwmActor, request.Release()); - } - } - - void SendToWorker(const TRequestInfo& node, TDuration backoff = TDuration()) { - auto nodeId = node.WorkerInfo.GetNodeId(); - auto request = MakeHolder<TEvAllocateWorkersRequest>(1, "", TMaybe<ui64>(node.ResourceId)); - if (Timeout) { - request->Record.SetFreeWorkerAfterMs(Timeout.MilliSeconds()); - } - request->Record.SetTraceId(TraceId); - if (!Tasks.empty()) { - request->Record.SetCreateComputeActor(true); - request->Record.SetComputeActorType(ComputeActorType); + } + } + + void SendToWorker(const TRequestInfo& node, TDuration backoff = TDuration()) { + auto nodeId = node.WorkerInfo.GetNodeId(); + auto request = MakeHolder<TEvAllocateWorkersRequest>(1, "", TMaybe<ui64>(node.ResourceId)); + if (Timeout) { + request->Record.SetFreeWorkerAfterMs(Timeout.MilliSeconds()); + } + request->Record.SetTraceId(TraceId); + if (!Tasks.empty()) { + request->Record.SetCreateComputeActor(true); + request->Record.SetComputeActorType(ComputeActorType); ActorIdToProto(ControlId, request->Record.MutableResultActorId()); - *request->Record.AddTask() = node.Task; - } - YQL_LOG(WARN) << "Send TEvAllocateWorkersRequest to " << NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(node.WorkerInfo, 0); - if (backoff) { - TActivationContext::Schedule(backoff, new IEventHandle( - MakeWorkerManagerActorID(nodeId), SelfId(), request.Release(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagGenerateUnsureUndelivered, - node.ResourceId)); - } else { - Send( - MakeWorkerManagerActorID(nodeId), - request.Release(), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagGenerateUnsureUndelivered, - node.ResourceId); - } - } - - void Fail(const ui64 cookie, const TString& reason) { - YQL_LOG_CTX_SCOPE(TraceId); - if (FailState) { - return; - } - auto maybeRequestInfo = RequestedNodes.find(cookie); - if (!Answered && maybeRequestInfo != RequestedNodes.end() && maybeRequestInfo->second.Retry < NetworkRetries) { - YQL_LOG(WARN) << "Retry Allocate Request"; - auto& requestInfo = RequestedNodes[cookie]; - requestInfo.Retry ++; - *RetryCounter += 1; - QueryStat.AddCounter("Retry", 1); - SendToWorker(requestInfo, TDuration::MilliSeconds(100)); - return; - } - FailState = true; - TString workerInfo; - - if (maybeRequestInfo != RequestedNodes.end()) { - workerInfo = NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(maybeRequestInfo->second.WorkerInfo, 0); - FailedWorkers.push_back(maybeRequestInfo->second.WorkerInfo); - - auto delta = TInstant::Now() - maybeRequestInfo->second.StartTime; - // catched at grpc_service - QueryStat.AddCounter( - QueryStat.GetCounterName("Actor", {{"ClusterName", maybeRequestInfo->second.ClusterName}}, "CreateFailTime"), - delta); - } - TString message = "Disconnected from worker: `" + workerInfo + "', reason: " + reason; - YQL_LOG(ERROR) << message; - auto response = MakeHolder<TEvAllocateWorkersResponse>(message); - QueryStat.FlushCounters(response->Record); - Send(SenderId, response.Release()); - } - + *request->Record.AddTask() = node.Task; + } + YQL_LOG(WARN) << "Send TEvAllocateWorkersRequest to " << NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(node.WorkerInfo, 0); + if (backoff) { + TActivationContext::Schedule(backoff, new IEventHandle( + MakeWorkerManagerActorID(nodeId), SelfId(), request.Release(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagGenerateUnsureUndelivered, + node.ResourceId)); + } else { + Send( + MakeWorkerManagerActorID(nodeId), + request.Release(), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagGenerateUnsureUndelivered, + node.ResourceId); + } + } + + void Fail(const ui64 cookie, const TString& reason) { + YQL_LOG_CTX_SCOPE(TraceId); + if (FailState) { + return; + } + auto maybeRequestInfo = RequestedNodes.find(cookie); + if (!Answered && maybeRequestInfo != RequestedNodes.end() && maybeRequestInfo->second.Retry < NetworkRetries) { + YQL_LOG(WARN) << "Retry Allocate Request"; + auto& requestInfo = RequestedNodes[cookie]; + requestInfo.Retry ++; + *RetryCounter += 1; + QueryStat.AddCounter("Retry", 1); + SendToWorker(requestInfo, TDuration::MilliSeconds(100)); + return; + } + FailState = true; + TString workerInfo; + + if (maybeRequestInfo != RequestedNodes.end()) { + workerInfo = NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(maybeRequestInfo->second.WorkerInfo, 0); + FailedWorkers.push_back(maybeRequestInfo->second.WorkerInfo); + + auto delta = TInstant::Now() - maybeRequestInfo->second.StartTime; + // catched at grpc_service + QueryStat.AddCounter( + QueryStat.GetCounterName("Actor", {{"ClusterName", maybeRequestInfo->second.ClusterName}}, "CreateFailTime"), + delta); + } + TString message = "Disconnected from worker: `" + workerInfo + "', reason: " + reason; + YQL_LOG(ERROR) << message; + auto response = MakeHolder<TEvAllocateWorkersResponse>(message); + QueryStat.FlushCounters(response->Record); + Send(SenderId, response.Release()); + } + const TActorId GwmActor; const TActorId SenderId; const TActorId ControlId; - - THashMap<ui64, TRequestInfo> RequestedNodes; - ui32 RequestedCount; + + THashMap<ui64, TRequestInfo> RequestedNodes; + ui32 RequestedCount; ui64 ResourceId = 0; - bool LocalMode = false; - - TVector<NDqProto::TWorkerGroup> AllocatedWorkers; + bool LocalMode = false; + + TVector<NDqProto::TWorkerGroup> AllocatedWorkers; ui32 AllocatedCount = 0; - - bool FailState = false; - bool Answered = false; - - TVector<Yql::DqsProto::TWorkerInfo> FailedWorkers; - - const TString TraceId; - const TDqConfiguration::TPtr Settings; - int NetworkRetries; - TDuration Timeout; - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TDynamicCounters::TCounterPtr RetryCounter; - - TCounters QueryStat; - - TVector<NYql::NDqProto::TDqTask> Tasks; // for compute actor - const TString ComputeActorType; -}; - -NActors::IActor* CreateResourceAllocator( + + bool FailState = false; + bool Answered = false; + + TVector<Yql::DqsProto::TWorkerInfo> FailedWorkers; + + const TString TraceId; + const TDqConfiguration::TPtr Settings; + int NetworkRetries; + TDuration Timeout; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + TDynamicCounters::TCounterPtr RetryCounter; + + TCounters QueryStat; + + TVector<NYql::NDqProto::TDqTask> Tasks; // for compute actor + const TString ComputeActorType; +}; + +NActors::IActor* CreateResourceAllocator( TActorId gwmActor, - TActorId senderId, + TActorId senderId, TActorId controlId, - ui32 size, - const TString& traceId, - const TDqConfiguration::TPtr& settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - const TVector<NYql::NDqProto::TDqTask>& tasks, - const TString& computeActorType) -{ - return new TResourceAllocator(gwmActor, senderId, controlId, size, traceId, settings, counters, tasks, computeActorType); -} - -} // namespace NYql + ui32 size, + const TString& traceId, + const TDqConfiguration::TPtr& settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TVector<NYql::NDqProto::TDqTask>& tasks, + const TString& computeActorType) +{ + return new TResourceAllocator(gwmActor, senderId, controlId, size, traceId, settings, counters, tasks, computeActorType); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/resource_allocator.h b/ydb/library/yql/providers/dq/actors/resource_allocator.h index 5fd15764bb..ab5db2df0e 100644 --- a/ydb/library/yql/providers/dq/actors/resource_allocator.h +++ b/ydb/library/yql/providers/dq/actors/resource_allocator.h @@ -1,20 +1,20 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/dq/proto/dq_tasks.pb.h> #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> - + #include <library/cpp/actors/core/actor.h> -#include <library/cpp/monlib/dynamic_counters/counters.h> - -namespace NYql { - NActors::IActor* CreateResourceAllocator( +#include <library/cpp/monlib/dynamic_counters/counters.h> + +namespace NYql { + NActors::IActor* CreateResourceAllocator( NActors::TActorId gwmActor, NActors::TActorId senderId, - NActors::TActorId resultId, - ui32 workerCount, - const TString& traceId, - const TDqConfiguration::TPtr& settings, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - const TVector<NYql::NDqProto::TDqTask>& tasks = {}, - const TString& computeActorType = "old"); -} // namespace NYql + NActors::TActorId resultId, + ui32 workerCount, + const TString& traceId, + const TDqConfiguration::TPtr& settings, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TVector<NYql::NDqProto::TDqTask>& tasks = {}, + const TString& computeActorType = "old"); +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp index 025f33048b..59ae95c3af 100644 --- a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp +++ b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp @@ -1,4 +1,4 @@ -#include "result_aggregator.h" +#include "result_aggregator.h" #include "result_receiver.h" #include "proto_builder.h" #include "full_result_writer.h" @@ -9,16 +9,16 @@ #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/dq/counters/counters.h> - + #include <ydb/library/yql/providers/dq/common/yql_dq_common.h> - + #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/sql/sql.h> #include <ydb/library/yql/utils/failure_injector/failure_injector.h> #include <ydb/library/yql/utils/actor_log/log.h> #include <ydb/library/yql/utils/log/log.h> - + #include <ydb/public/lib/yson_value/ydb_yson_value.h> #include <library/cpp/actors/core/actorsystem.h> @@ -36,15 +36,15 @@ #include <util/stream/str.h> #include <util/stream/length.h> - + namespace NYql::NDqs::NExecutionHelpers { - + using namespace NYql; using namespace NYql::NDqProto; using namespace NYql::NNodes; - + using namespace NKikimr::NMiniKQL; - + using namespace Yql::DqsProto; using namespace NYql::NDq; @@ -53,13 +53,13 @@ using namespace NYql::NDqProto; using namespace NActors; namespace { - + class TResultAggregator: public TSynchronizableRichActor<TResultAggregator>, NYql::TCounters { static constexpr ui32 MAX_RESULT_BATCH = 2048; public: static constexpr char ActorName[] = "YQL_DQ_RESULT_AGGREGATOR"; - + explicit TResultAggregator(const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId, const TDqConfiguration::TPtr& settings, const TString& resultType, NActors::TActorId graphExecutionEventsId, bool discard) : TSynchronizableRichActor<TResultAggregator>(&TResultAggregator::Handler) @@ -77,7 +77,7 @@ public: PullRequestTimeout = TDuration::MilliSeconds(settings->PullRequestTimeoutMs.Get().GetOrElse(0)); PingTimeout = TDuration::MilliSeconds(settings->PingTimeoutMs.Get().GetOrElse(0)); PingPeriod = Max(PingTimeout/4, TDuration::MilliSeconds(1000)); - + SizeLimit = Settings->_AllResultsBytesLimit.Get().GetOrElse(64000000); YQL_LOG(DEBUG) << "_AllResultsBytesLimit = " << SizeLimit; @@ -89,7 +89,7 @@ public: } private: -#define HANDLER_STUB(TEvType) \ +#define HANDLER_STUB(TEvType) \ cFunc(TEvType::EventType, [this]() { \ YQL_LOG_CTX_SCOPE(TraceId); \ YQL_LOG(DEBUG) << "Unexpected event " << ( #TEvType ); \ @@ -151,28 +151,28 @@ private: if (PingTimeout && now - PingStartTime > PingTimeout) { OnError("PingTimeout " + ToString(SourceID.NodeId()), true, true); } - + if (!PingRequested) { PingRequested = true; PingStartTime = now; Send(SourceID, MakeHolder<TEvPingRequest>(), IEventHandle::FlagTrackDelivery); } - + TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get()); } - + void OnReadyState(TEvReadyState::TPtr& ev, const TActorContext&) { YQL_LOG_CTX_SCOPE(TraceId); AddCounters(ev->Get()->Record); - + SourceID = NActors::ActorIdFromProto(ev->Get()->Record.GetSourceId()); Send(SelfId(), MakeHolder<TEvPullResult>()); - + PingStartTime = PullRequestStartTime = TInstant::Now(); TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way()); Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get()); - + AddCriticalEventType(TEvents::TEvWakeup::EventType); AddCriticalEventType(TEvPingResponse::EventType); } @@ -196,10 +196,10 @@ private: void OnPingResponse(TEvPingResponse::TPtr&, const TActorContext&) { PingRequested = false; } - + void OnPullResponse(TEvPullDataResponse::TPtr& ev, const TActorContext&) { YQL_LOG_CTX_SCOPE(TraceId); - + if (FinishCalled) { // finalization has been begun, actor will not kill himself anymore, should ignore responses instead return; @@ -208,7 +208,7 @@ private: auto& response = ev->Get()->Record; AddCounters(response); - + switch (response.GetResponseType()) { case NYql::NDqProto::CONTINUE: { Send(SelfId(), MakeHolder<TEvPullResult>()); @@ -231,7 +231,7 @@ private: YQL_ENSURE(false, "Unknown pull result"); break; } - + if (!Discard) { auto fullResultTableEnabled = Settings->EnableFullResultWrite.Get().GetOrElse(false); @@ -239,7 +239,7 @@ private: WriteToFullResultTable(new NDqProto::TData(std::move(*response.MutableData()))); } else { DataParts.emplace_back(std::move(*response.MutableData())); - + bool full = true; bool exceedRows = false; try { @@ -258,7 +258,7 @@ private: } catch (...) { OnError(CurrentExceptionMessage(), false, false); return; - } + } if (!full) { if (fullResultTableEnabled) { @@ -311,7 +311,7 @@ private: // TODO Customize return FullResultSentBytes - FullResultReceivedBytes <= 32_MB; } - + template <class TCallback> void UpdateEventQueueStatus(TCallback callback) { YQL_LOG(DEBUG) << "UpdateEQStatus before: sent " << (FullResultSentBytes / 1024.0) << " kB " @@ -367,7 +367,7 @@ private: Send(FullResultWriterID, MakeHolder<TEvPullDataResponse>(response)); FullResultSentBytes += respSize; } - + void Finish(bool truncated = false, const TIssues& issues = {}) { YQL_LOG(DEBUG) << __FUNCTION__ << ", truncated=" << truncated; YQL_ENSURE(!FinishCalled); @@ -386,7 +386,7 @@ private: void DoFinish() { Send(ExecuterID, new TEvGraphFinished()); } - + void OnFullResultWriterResponse(TEvDqFailure::TPtr& ev, const TActorContext&) { YQL_LOG_CTX_SCOPE(TraceId); YQL_LOG(DEBUG) << __FUNCTION__; @@ -396,7 +396,7 @@ private: Send(ExecuterID, ev->Release().Release()); } } - + void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { YQL_LOG_CTX_SCOPE(TraceId); YQL_ENSURE(!ev->Get()->Record.HasResultSet() && ev->Get()->Record.GetYson().empty()); @@ -500,7 +500,7 @@ public: private: STRICT_STFUNC(Handler, { HFunc(TEvQueryResponse, OnQueryResult); }) - + void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { if (!ev->Get()->Record.HasResultSet()&&ev->Get()->Record.GetYson().empty()) { NYql::TIssues issues; diff --git a/ydb/library/yql/providers/dq/actors/result_aggregator.h b/ydb/library/yql/providers/dq/actors/result_aggregator.h index 7958bb589b..450bc2de88 100644 --- a/ydb/library/yql/providers/dq/actors/result_aggregator.h +++ b/ydb/library/yql/providers/dq/actors/result_aggregator.h @@ -8,13 +8,13 @@ #include <library/cpp/actors/core/actor.h> namespace NYql::NDqs::NExecutionHelpers { - THolder<NActors::IActor> MakeResultAggregator( + THolder<NActors::IActor> MakeResultAggregator( const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId, - const THashMap<TString, TString>& secureParams, - const TDqConfiguration::TPtr& settings, + const THashMap<TString, TString>& secureParams, + const TDqConfiguration::TPtr& settings, const TString& resultType, bool discard, const NActors::TActorId& graphExecutionEventsId); -} // namespace NYql::NDqs::NExecutionHelpers +} // namespace NYql::NDqs::NExecutionHelpers diff --git a/ydb/library/yql/providers/dq/actors/result_receiver.cpp b/ydb/library/yql/providers/dq/actors/result_receiver.cpp index a06cb92610..7fdb73f257 100644 --- a/ydb/library/yql/providers/dq/actors/result_receiver.cpp +++ b/ydb/library/yql/providers/dq/actors/result_receiver.cpp @@ -3,7 +3,7 @@ #include <ydb/library/yql/providers/dq/actors/execution_helpers.h> #include <ydb/library/yql/providers/dq/actors/events.h> - + #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> #include <ydb/library/yql/providers/dq/actors/executer_actor.h> #include <ydb/library/yql/providers/dq/counters/counters.h> @@ -11,72 +11,72 @@ #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> - + #include <ydb/public/lib/yson_value/ydb_yson_value.h> - -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/actors/core/event_pb.h> -#include <library/cpp/actors/core/executor_pool_basic.h> -#include <library/cpp/actors/core/hfunc.h> -#include <library/cpp/actors/core/scheduler_basic.h> -#include <library/cpp/threading/future/future.h> -#include <library/cpp/protobuf/util/pb_io.h> - -#include <util/generic/size_literals.h> -#include <util/generic/ptr.h> -#include <util/string/split.h> -#include <util/system/types.h> - -namespace NYql { - -using namespace NActors; -using namespace NDqs; - -namespace { - -class TResultReceiver: public TRichActor<TResultReceiver> { -public: + +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/event_pb.h> +#include <library/cpp/actors/core/executor_pool_basic.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/scheduler_basic.h> +#include <library/cpp/threading/future/future.h> +#include <library/cpp/protobuf/util/pb_io.h> + +#include <util/generic/size_literals.h> +#include <util/generic/ptr.h> +#include <util/string/split.h> +#include <util/system/types.h> + +namespace NYql { + +using namespace NActors; +using namespace NDqs; + +namespace { + +class TResultReceiver: public TRichActor<TResultReceiver> { +public: static constexpr char ActorName[] = "YQL_DQ_RESULT_RECEIVER"; - + explicit TResultReceiver(const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId, const TDqConfiguration::TPtr& settings, const THashMap<TString, TString>& secureParams, const TString& resultType, bool discard) - : TRichActor<TResultReceiver>(&TResultReceiver::Handler) + : TRichActor<TResultReceiver>(&TResultReceiver::Handler) , ExecuterId(executerId) - , TraceId(traceId) - , Settings(settings) - , SecureParams(std::move(secureParams)) - , ResultBuilder( - resultType + , TraceId(traceId) + , Settings(settings) + , SecureParams(std::move(secureParams)) + , ResultBuilder( + resultType ? MakeHolder<TProtoBuilder>(resultType, columns) - : nullptr) + : nullptr) , Discard(discard) - { - if (Settings) { - if (Settings->_AllResultsBytesLimit.Get()) { - YQL_LOG(DEBUG) << "_AllResultsBytesLimit = " << *Settings->_AllResultsBytesLimit.Get(); - } - if (Settings->_RowsLimitPerWrite.Get()) { - YQL_LOG(DEBUG) << "_RowsLimitPerWrite = " << *Settings->_RowsLimitPerWrite.Get(); - } - } - - Y_UNUSED(Size); - Y_UNUSED(Rows); - } - -private: - STRICT_STFUNC(Handler, { + { + if (Settings) { + if (Settings->_AllResultsBytesLimit.Get()) { + YQL_LOG(DEBUG) << "_AllResultsBytesLimit = " << *Settings->_AllResultsBytesLimit.Get(); + } + if (Settings->_RowsLimitPerWrite.Get()) { + YQL_LOG(DEBUG) << "_RowsLimitPerWrite = " << *Settings->_RowsLimitPerWrite.Get(); + } + } + + Y_UNUSED(Size); + Y_UNUSED(Rows); + } + +private: + STRICT_STFUNC(Handler, { HFunc(NDq::TEvDqCompute::TEvChannelData, OnChannelData) - HFunc(TEvReadyState, OnReadyState); - HFunc(TEvQueryResponse, OnQueryResult); - cFunc(TEvents::TEvPoison::EventType, PassAway) - }) - + HFunc(TEvReadyState, OnReadyState); + HFunc(TEvQueryResponse, OnQueryResult); + cFunc(TEvents::TEvPoison::EventType, PassAway) + }) + void OnChannelData(NDq::TEvDqCompute::TEvChannelData::TPtr& ev, const TActorContext&) { - YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG_CTX_SCOPE(TraceId); auto res = MakeHolder<NDq::TEvDqCompute::TEvChannelDataAck>(); - + if (!Discard) { if (!Finished && ev->Get()->Record.GetChannelData().GetData().GetRaw().size() > 0) { DataParts.emplace_back(std::move(ev->Get()->Record.GetChannelData().GetData())); @@ -85,7 +85,7 @@ private: YQL_LOG(DEBUG) << "Size: " << Size; YQL_LOG(DEBUG) << "Rows: " << Rows; } - + if (Size > 64000000 /* grpc limit*/) { OnError("Too big result (grpc limit reached: " + ToString(Size) + " > 64000000)" , false, true); } else if (Settings && Settings->_AllResultsBytesLimit.Get() && Size > *Settings->_AllResultsBytesLimit.Get()) { @@ -99,84 +99,84 @@ private: Issues.AddIssue(issue); Finish(/*truncated = */ true); } - } - - res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); - res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); - res->Record.SetFreeSpace(256_MB); - res->Record.SetFinish(Finished); - Send(ev->Sender, res.Release()); - - YQL_LOG(DEBUG) << "Finished: " << ev->Get()->Record.GetChannelData().GetFinished(); - } - + } + + res->Record.SetChannelId(ev->Get()->Record.GetChannelData().GetChannelId()); + res->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); + res->Record.SetFreeSpace(256_MB); + res->Record.SetFinish(Finished); + Send(ev->Sender, res.Release()); + + YQL_LOG(DEBUG) << "Finished: " << ev->Get()->Record.GetChannelData().GetFinished(); + } + void OnReadyState(TEvReadyState::TPtr&, const TActorContext&) { // do nothing - } - + } + void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) { NDqProto::TQueryResponse result(ev->Get()->Record); YQL_ENSURE(!result.HasResultSet() && result.GetYson().empty()); - if (ResultBuilder) { + if (ResultBuilder) { try { TString yson = Discard ? "" : ResultBuilder->BuildYson( - DataParts, - Settings && Settings->_AllResultsBytesLimit.Get() - ? *Settings->_AllResultsBytesLimit.Get() - : 64000000 /* grpc limit*/); - *result.MutableYson() = yson; + DataParts, + Settings && Settings->_AllResultsBytesLimit.Get() + ? *Settings->_AllResultsBytesLimit.Get() + : 64000000 /* grpc limit*/); + *result.MutableYson() = yson; } catch (...) { - Issues.AddIssue(TIssue(CurrentExceptionMessage()).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)); - result.SetNeedFallback(true); - } - } else { - if (Rows > 0) { + Issues.AddIssue(TIssue(CurrentExceptionMessage()).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)); + result.SetNeedFallback(true); + } + } else { + if (Rows > 0) { Issues.AddIssue(TIssue("Non empty rows: " + ToString(Rows)).SetCode(0, TSeverityIds::S_WARNING)); - } - } + } + } if (!Issues.Empty()) { IssuesToMessage(Issues, result.MutableIssues()); - } + } result.SetTruncated(Truncated); Send(ExecuterId, new TEvQueryResponse(std::move(result))); - } - - void OnError(const TString& message, bool retriable, bool needFallback) { - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "OnError " << message; + } + + void OnError(const TString& message, bool retriable, bool needFallback) { + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "OnError " << message; auto req = MakeHolder<TEvDqFailure>(TIssue(message).SetCode(-1, TSeverityIds::S_ERROR), retriable, needFallback); Send(ExecuterId, req.Release()); - Finished = true; - } - + Finished = true; + } + void Finish(bool truncated = false) { Send(ExecuterId, new TEvGraphFinished()); - Finished = true; + Finished = true; Truncated = truncated; - } - + } + const NActors::TActorId ExecuterId; - TVector<NDqProto::TData> DataParts; - const TString TraceId; - TDqConfiguration::TPtr Settings; - bool Finished = false; + TVector<NDqProto::TData> DataParts; + const TString TraceId; + TDqConfiguration::TPtr Settings; + bool Finished = false; bool Truncated = false; TIssues Issues; - ui64 Size = 0; - ui64 Rows = 0; + ui64 Size = 0; + ui64 Rows = 0; // const Yql::DqsProto::TFullResultTable FullResultTable; - const THashMap<TString, TString> SecureParams; + const THashMap<TString, TString> SecureParams; // THolder<IFullResultWriter> FullResultWriter; - THolder<TProtoBuilder> ResultBuilder; + THolder<TProtoBuilder> ResultBuilder; bool Discard = false; -}; - -} /* namespace */ - +}; + +} /* namespace */ + THolder<NActors::IActor> MakeResultReceiver(const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId, const TDqConfiguration::TPtr& settings, const THashMap<TString, TString>& secureParams, const TString& resultType, bool discard) { return MakeHolder<TResultReceiver>(columns, executerId, traceId, settings, secureParams, resultType, discard); -} - -} /* namespace NYql */ +} + +} /* namespace NYql */ diff --git a/ydb/library/yql/providers/dq/actors/result_receiver.h b/ydb/library/yql/providers/dq/actors/result_receiver.h index 19a2c3b1b8..968bcede10 100644 --- a/ydb/library/yql/providers/dq/actors/result_receiver.h +++ b/ydb/library/yql/providers/dq/actors/result_receiver.h @@ -7,13 +7,13 @@ namespace NYql { -THolder<NActors::IActor> MakeResultReceiver( - const TVector<TString>& columns, +THolder<NActors::IActor> MakeResultReceiver( + const TVector<TString>& columns, const NActors::TActorId& executerId, - const TString& traceId, - const TDqConfiguration::TPtr& settings, + const TString& traceId, + const TDqConfiguration::TPtr& settings, // const Yql::DqsProto::TFullResultTable& resultTable, - const THashMap<TString, TString>& secureParams, + const THashMap<TString, TString>& secureParams, const TString& resultBuilder, bool discard ); diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp index a5d8916b06..238828e453 100644 --- a/ydb/library/yql/providers/dq/actors/task_controller.cpp +++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp @@ -1,9 +1,9 @@ #include "task_controller.h" -#include "execution_helpers.h" -#include "events.h" +#include "execution_helpers.h" +#include "events.h" #include "proto_builder.h" -#include "actor_helpers.h" -#include "executer_actor.h" +#include "actor_helpers.h" +#include "executer_actor.h" #include <ydb/library/yql/providers/dq/counters/counters.h> @@ -138,8 +138,8 @@ private: } case NDqProto::COMPUTE_STATE_FAILURE: { // TODO: don't convert issues to string - NYql::IssuesFromMessage(state.GetIssues(), Issues); - OnError(Issues.ToString(), false, false); + NYql::IssuesFromMessage(state.GetIssues(), Issues); + OnError(Issues.ToString(), false, false); break; } case NDqProto::COMPUTE_STATE_EXECUTING: { @@ -526,7 +526,7 @@ private: NYq::NCommon::TServiceCounters ServiceCounters; TDuration PingPeriod = TDuration::Zero(); TDuration AggrPeriod = TDuration::Zero(); - TIssues Issues; + TIssues Issues; ui64 PingCookie = 0; }; diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp index 86672f187d..f4503a60b2 100644 --- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp +++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp @@ -6,527 +6,527 @@ #include <ydb/library/yql/utils/failure_injector/failure_injector.h> #include <ydb/library/yql/utils/actor_log/log.h> #include <ydb/library/yql/utils/log/log.h> - + #include <ydb/library/yql/minikql/mkql_string_util.h> - + #include <library/cpp/actors/core/event_pb.h> #include <library/cpp/actors/core/hfunc.h> -#include <util/stream/file.h> -#include <util/string/split.h> +#include <util/stream/file.h> +#include <util/string/split.h> #include <util/stream/output.h> - + using namespace NYql::NDq; -using namespace NYql::NDq::NTaskRunnerActor; +using namespace NYql::NDq::NTaskRunnerActor; using namespace NYql::NDqProto; -using namespace NActors; +using namespace NActors; namespace NYql::NDqs { -struct TInputChannel { - bool Finished{false}; - bool Requested{false}; - ui32 InputId = 0; - ui32 ChannelId = 0; - NActors::TActorId ActorID; - int Retries = 0; - TInstant RequestTime; - TInstant ResponseTime; - - bool PingRequested{false}; - TInstant PingStartTime; -}; - -struct TOutputChannel { - ui32 OutputId = 0; - ui32 ChannelId = 0; - NActors::TActorId ActorID; - bool Finished = false; - TInstant RequestTime = TInstant(); -}; - -struct TSourceInfo { - IDqSourceActor* SourceActor = nullptr; - NActors::IActor* Actor = nullptr; - i64 FreeSpace = 1; - bool HasData = false; - bool PushStarted = false; - bool Finished = false; - NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr; -}; - -struct TSinkInfo { - IDqSinkActor* SinkActor = nullptr; - NActors::IActor* Actor = nullptr; - bool Finished = false; - NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr; -}; - -class TDqWorker: public TRichActor<TDqWorker> - , IDqSourceActor::ICallbacks - , IDqSinkActor::ICallbacks - , ITaskRunnerActor::ICallbacks -{ - static constexpr ui32 INPUT_SIZE = 100000; - -public: +struct TInputChannel { + bool Finished{false}; + bool Requested{false}; + ui32 InputId = 0; + ui32 ChannelId = 0; + NActors::TActorId ActorID; + int Retries = 0; + TInstant RequestTime; + TInstant ResponseTime; + + bool PingRequested{false}; + TInstant PingStartTime; +}; + +struct TOutputChannel { + ui32 OutputId = 0; + ui32 ChannelId = 0; + NActors::TActorId ActorID; + bool Finished = false; + TInstant RequestTime = TInstant(); +}; + +struct TSourceInfo { + IDqSourceActor* SourceActor = nullptr; + NActors::IActor* Actor = nullptr; + i64 FreeSpace = 1; + bool HasData = false; + bool PushStarted = false; + bool Finished = false; + NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr; +}; + +struct TSinkInfo { + IDqSinkActor* SinkActor = nullptr; + NActors::IActor* Actor = nullptr; + bool Finished = false; + NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr; +}; + +class TDqWorker: public TRichActor<TDqWorker> + , IDqSourceActor::ICallbacks + , IDqSinkActor::ICallbacks + , ITaskRunnerActor::ICallbacks +{ + static constexpr ui32 INPUT_SIZE = 100000; + +public: static constexpr char ActorName[] = "YQL_DQ_WORKER"; - explicit TDqWorker( - const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, - const IDqSourceActorFactory::TPtr& sourceActorFactory, - const IDqSinkActorFactory::TPtr& sinkActorFactory, - TWorkerRuntimeData* runtimeData, - const TString& traceId) - : TRichActor<TDqWorker>(&TDqWorker::Handler) - , SourceActorFactory(sourceActorFactory) - , SinkActorFactory(sinkActorFactory) - , TaskRunnerActorFactory(taskRunnerActorFactory) - , RuntimeData(runtimeData) - , TraceId(traceId) + explicit TDqWorker( + const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, + const IDqSourceActorFactory::TPtr& sourceActorFactory, + const IDqSinkActorFactory::TPtr& sinkActorFactory, + TWorkerRuntimeData* runtimeData, + const TString& traceId) + : TRichActor<TDqWorker>(&TDqWorker::Handler) + , SourceActorFactory(sourceActorFactory) + , SinkActorFactory(sinkActorFactory) + , TaskRunnerActorFactory(taskRunnerActorFactory) + , RuntimeData(runtimeData) + , TraceId(traceId) { - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "TDqWorker created "; - - if (RuntimeData) { - RuntimeData->OnWorkerStart(TraceId); - } + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "TDqWorker created "; + + if (RuntimeData) { + RuntimeData->OnWorkerStart(TraceId); + } } - ~TDqWorker() - { - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "TDqWorker destroyed "; - - if (RuntimeData) { - RuntimeData->OnWorkerStop(TraceId); - } - } - - void DoPassAway() override { - YQL_LOG_CTX_SCOPE(TraceId); + ~TDqWorker() + { + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "TDqWorker destroyed "; + + if (RuntimeData) { + RuntimeData->OnWorkerStop(TraceId); + } + } + + void DoPassAway() override { + YQL_LOG_CTX_SCOPE(TraceId); for (const auto& inputs : InputMap) { Send(inputs.first, new NActors::TEvents::TEvPoison()); } - YQL_LOG(DEBUG) << "TDqWorker passed away "; - if (Actor) { - Actor->PassAway(); - } - for (const auto& [_, v] : SourcesMap) { - v.SourceActor->PassAway(); - } - for (const auto& [_, v] : SinksMap) { - v.SinkActor->PassAway(); - } - Dump(); + YQL_LOG(DEBUG) << "TDqWorker passed away "; + if (Actor) { + Actor->PassAway(); + } + for (const auto& [_, v] : SourcesMap) { + v.SourceActor->PassAway(); + } + for (const auto& [_, v] : SinksMap) { + v.SinkActor->PassAway(); + } + Dump(); } -private: - STRICT_STFUNC(Handler, { - HFunc(TEvDqTask, OnDqTask); - HFunc(TEvPullDataRequest, OnPullRequest); - HFunc(TEvPullDataResponse, OnPullResponse); - HFunc(TEvPingRequest, OnPingRequest); - HFunc(TEvPingResponse, OnPingResponse); - HFunc(TEvents::TEvUndelivered, OnUndelivered); - cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); - - HFunc(TEvTaskRunnerCreateFinished, OnTaskRunnerCreated); - HFunc(TEvChannelPopFinished, OnChannelPopFinished); - HFunc(TEvTaskRunFinished, OnRunFinished); - HFunc(TEvSourcePushFinished, OnSourcePushFinished); - - HFunc(TEvError, OnErrorFromPipe); - HFunc(TEvContinueRun, OnContinueRun); - cFunc(TEvents::TEvWakeup::EventType, OnWakeup); +private: + STRICT_STFUNC(Handler, { + HFunc(TEvDqTask, OnDqTask); + HFunc(TEvPullDataRequest, OnPullRequest); + HFunc(TEvPullDataResponse, OnPullResponse); + HFunc(TEvPingRequest, OnPingRequest); + HFunc(TEvPingResponse, OnPingResponse); + HFunc(TEvents::TEvUndelivered, OnUndelivered); + cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); + + HFunc(TEvTaskRunnerCreateFinished, OnTaskRunnerCreated); + HFunc(TEvChannelPopFinished, OnChannelPopFinished); + HFunc(TEvTaskRunFinished, OnRunFinished); + HFunc(TEvSourcePushFinished, OnSourcePushFinished); + + HFunc(TEvError, OnErrorFromPipe); + HFunc(TEvContinueRun, OnContinueRun); + cFunc(TEvents::TEvWakeup::EventType, OnWakeup); }) - TString ParseStatus(const TString& input, bool* retriableFlag, bool* fallbackFlag) { - TString result; - for (TStringBuf line: StringSplitter(input).SplitByString("\n").SkipEmpty()) { - if (line.StartsWith("Counter1:")) { - TVector<TString> parts; - Split(TString(line), " ", parts); - if (parts.size() >= 3) { - auto name = parts[1]; - i64 value; - if (TryFromString<i64>(parts[2], value)) { - Stat.AddCounter(name, TDuration::MilliSeconds(value)); - } - } - } else if (line.StartsWith("Counter:")) { - TVector<TString> parts; - Split(TString(line), " ", parts); - // name sum min max avg count - if (parts.size() >= 7) { - auto name = parts[1]; - TCounters::TEntry entry; - if ( - TryFromString<i64>(parts[2], entry.Sum) && - TryFromString<i64>(parts[3], entry.Min) && - TryFromString<i64>(parts[4], entry.Max) && - TryFromString<i64>(parts[5], entry.Avg) && - TryFromString<i64>(parts[6], entry.Count)) - { - Stat.AddCounter(name, entry); - } - } - } else if (line.Contains("mlockall failed")) { - // skip - } else { - if (fallbackFlag && !*fallbackFlag) { - if (line.Contains("FindColumnInfo(): requirement memberType->GetKind() == TType::EKind::Data")) { - // temporary workaround for part6/produce-reduce_lambda_list_table-default.txt - *fallbackFlag = true; - } else if (line.Contains("Unsupported builtin function:")) { - // temporary workaround for YQL-11791 - *fallbackFlag = true; - } else if (line.Contains("embedded:Len")) { - *fallbackFlag = true; - } else if (line.Contains("Container killed by OOM")) { - // temporary workaround for YQL-12066 - *fallbackFlag = true; + TString ParseStatus(const TString& input, bool* retriableFlag, bool* fallbackFlag) { + TString result; + for (TStringBuf line: StringSplitter(input).SplitByString("\n").SkipEmpty()) { + if (line.StartsWith("Counter1:")) { + TVector<TString> parts; + Split(TString(line), " ", parts); + if (parts.size() >= 3) { + auto name = parts[1]; + i64 value; + if (TryFromString<i64>(parts[2], value)) { + Stat.AddCounter(name, TDuration::MilliSeconds(value)); + } + } + } else if (line.StartsWith("Counter:")) { + TVector<TString> parts; + Split(TString(line), " ", parts); + // name sum min max avg count + if (parts.size() >= 7) { + auto name = parts[1]; + TCounters::TEntry entry; + if ( + TryFromString<i64>(parts[2], entry.Sum) && + TryFromString<i64>(parts[3], entry.Min) && + TryFromString<i64>(parts[4], entry.Max) && + TryFromString<i64>(parts[5], entry.Avg) && + TryFromString<i64>(parts[6], entry.Count)) + { + Stat.AddCounter(name, entry); + } + } + } else if (line.Contains("mlockall failed")) { + // skip + } else { + if (fallbackFlag && !*fallbackFlag) { + if (line.Contains("FindColumnInfo(): requirement memberType->GetKind() == TType::EKind::Data")) { + // temporary workaround for part6/produce-reduce_lambda_list_table-default.txt + *fallbackFlag = true; + } else if (line.Contains("Unsupported builtin function:")) { + // temporary workaround for YQL-11791 + *fallbackFlag = true; + } else if (line.Contains("embedded:Len")) { + *fallbackFlag = true; + } else if (line.Contains("Container killed by OOM")) { + // temporary workaround for YQL-12066 + *fallbackFlag = true; } else if (line.Contains("Expected data or optional of data, actual:")) { - // temporary workaround for YQL-12835 - *fallbackFlag = true; + // temporary workaround for YQL-12835 + *fallbackFlag = true; } else if (line.Contains("Cannot create Skiff writer for ")) { // temporary workaround for YQL-12986 *fallbackFlag = true; - } else if (line.Contains("Skiff format expected")) { - // temporary workaround for YQL-12986 - *fallbackFlag = true; + } else if (line.Contains("Skiff format expected")) { + // temporary workaround for YQL-12986 + *fallbackFlag = true; } else if (line.Contains("Pattern nodes can not get computation node by index:")) { // temporary workaround for YQL-12987 *fallbackFlag = true; } else if (line.Contains("contrib/libs/protobuf/src/google/protobuf/messagext.cc") && line.Contains("Message size") && line.Contains("exceeds")) { // temporary workaround for YQL-12988 *fallbackFlag = true; - } else if (line.Contains("Cannot start container")) { - // temporary workaround for YQL-14221 - *retriableFlag = true; - *fallbackFlag = true; - } else if (line.Contains("Cannot execl")) { - // YQL-14099 - *retriableFlag = true; - *fallbackFlag = true; - } else { - for (const auto& part : FallbackOn) { - if (line.Contains(part)) { - *fallbackFlag = true; - } - } - } - } - - result += line; - result += "\n"; - } - } - return result; - } - - void OnError(TString message = "", bool retriable = false, bool fallback = false, TMaybe<TEvError::TStatus> status = Nothing()) { - if (!Executer) { - // Posible Error on Undelivered before OnDqTask - YQL_LOG(ERROR) << "Error " << message; - return; - } - - int exitCode = -1; - TString stderrStr; - - if (status) { - exitCode = status->ExitCode; - stderrStr = TStringBuilder{} - << "ExitCode: " << exitCode << "\n" - << "StageId: " << StageId << "\n" - << ParseStatus(status->Stderr, &retriable, &fallback); - } - - if (message.Empty() && (status.Empty() || exitCode == 0)) { - message = CurrentExceptionMessage(); - } - - auto issueCode = fallback - ? TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR - : TIssuesIds::DQ_GATEWAY_ERROR; - - TIssue issue = TIssue(message).SetCode(issueCode, TSeverityIds::S_ERROR); - if (status && exitCode != 0) { - issue = TIssue(stderrStr).SetCode(issueCode, TSeverityIds::S_ERROR); - TStringBuf message = stderrStr; - auto parsedPos = TryParseTerminationMessage(message); - if (message.size() < stderrStr.size()) { - issue.AddSubIssue( - MakeIntrusive<TIssue>( - YqlIssue(parsedPos.GetOrElse(TPosition()), - TIssuesIds::DQ_GATEWAY_ERROR, - TString{message}))); - } - } - - auto req = MakeHolder<TEvDqFailure>(issue, retriable, fallback); - Stat.FlushCounters(req->Record); - Send(Executer, req.Release()); - } - - void OnErrorFromPipe(TEvError::TPtr& ev, const TActorContext& ) - { - OnError(ev->Get()->Message, ev->Get()->Retriable, ev->Get()->Fallback, ev->Get()->Status); - } - - void OnContinueRun(TEvContinueRun::TPtr&, const TActorContext& ctx) { - Run(ctx); - } - - void OnDqTask(TEvDqTask::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "TDqWorker::OnDqTask"; - + } else if (line.Contains("Cannot start container")) { + // temporary workaround for YQL-14221 + *retriableFlag = true; + *fallbackFlag = true; + } else if (line.Contains("Cannot execl")) { + // YQL-14099 + *retriableFlag = true; + *fallbackFlag = true; + } else { + for (const auto& part : FallbackOn) { + if (line.Contains(part)) { + *fallbackFlag = true; + } + } + } + } + + result += line; + result += "\n"; + } + } + return result; + } + + void OnError(TString message = "", bool retriable = false, bool fallback = false, TMaybe<TEvError::TStatus> status = Nothing()) { + if (!Executer) { + // Posible Error on Undelivered before OnDqTask + YQL_LOG(ERROR) << "Error " << message; + return; + } + + int exitCode = -1; + TString stderrStr; + + if (status) { + exitCode = status->ExitCode; + stderrStr = TStringBuilder{} + << "ExitCode: " << exitCode << "\n" + << "StageId: " << StageId << "\n" + << ParseStatus(status->Stderr, &retriable, &fallback); + } + + if (message.Empty() && (status.Empty() || exitCode == 0)) { + message = CurrentExceptionMessage(); + } + + auto issueCode = fallback + ? TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR + : TIssuesIds::DQ_GATEWAY_ERROR; + + TIssue issue = TIssue(message).SetCode(issueCode, TSeverityIds::S_ERROR); + if (status && exitCode != 0) { + issue = TIssue(stderrStr).SetCode(issueCode, TSeverityIds::S_ERROR); + TStringBuf message = stderrStr; + auto parsedPos = TryParseTerminationMessage(message); + if (message.size() < stderrStr.size()) { + issue.AddSubIssue( + MakeIntrusive<TIssue>( + YqlIssue(parsedPos.GetOrElse(TPosition()), + TIssuesIds::DQ_GATEWAY_ERROR, + TString{message}))); + } + } + + auto req = MakeHolder<TEvDqFailure>(issue, retriable, fallback); + Stat.FlushCounters(req->Record); + Send(Executer, req.Release()); + } + + void OnErrorFromPipe(TEvError::TPtr& ev, const TActorContext& ) + { + OnError(ev->Get()->Message, ev->Get()->Retriable, ev->Get()->Fallback, ev->Get()->Status); + } + + void OnContinueRun(TEvContinueRun::TPtr&, const TActorContext& ctx) { + Run(ctx); + } + + void OnDqTask(TEvDqTask::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "TDqWorker::OnDqTask"; + TFailureInjector::Reach("dq_task_failure", [] {::_exit(1); }); - Y_VERIFY(!TaskRunnerActor); - - Stat.StartCounter(Stat.GetCounterName("Actor", {{"ClusterName", RuntimeData ? RuntimeData->ClusterName : "local"}}, "ProcessInit")); + Y_VERIFY(!TaskRunnerActor); + + Stat.StartCounter(Stat.GetCounterName("Actor", {{"ClusterName", RuntimeData ? RuntimeData->ClusterName : "local"}}, "ProcessInit")); Y_VERIFY(!Executer); - Executer = ev->Sender; - Task = ev->Get()->Record.GetTask(); - - Yql::DqsProto::TTaskMeta taskMeta; - Task.GetMeta().UnpackTo(&taskMeta); - for (const auto& w : taskMeta.GetWorkerInfo()) { - AllWorkers.push_back(w); - } - StageId = taskMeta.GetStageId(); - Settings->Dispatch(taskMeta.GetSettings()); - Settings->FreezeDefaults(); - PullRequestTimeout = TDuration::MilliSeconds(Settings->PullRequestTimeoutMs.Get().GetOrElse(0)); - PingTimeout = TDuration::MilliSeconds(Settings->PingTimeoutMs.Get().GetOrElse(0)); - if (PingTimeout) { - PingPeriod = Max(PingTimeout/4, TDuration::MilliSeconds(1000)); - } - if (Settings->_FallbackOnRuntimeErrors.Get()) { - TString parts = Settings->_FallbackOnRuntimeErrors.Get().GetOrElse(""); - for (const auto& it : StringSplitter(parts).Split(',')) { - FallbackOn.insert(TString(it.Token())); - } - } - - NActors::IActor* actor; - std::tie(Actor, actor) = TaskRunnerActorFactory->Create(this, TraceId); - TaskRunnerActor = RegisterLocalChild(actor); - TDqTaskRunnerMemoryLimits limits; // used for local mode only - limits.ChannelBufferSize = 20_MB; - limits.OutputChunkMaxSize = 2_MB; - Send(TaskRunnerActor, new TEvTaskRunnerCreate(std::move(ev->Get()->Record.GetTask()), limits)); - } - - void OnTaskRunnerCreated(TEvTaskRunnerCreateFinished::TPtr& ev, const TActorContext& ) { - Stat.FlushCounter(Stat.GetCounterName("Actor", {{"ClusterName", RuntimeData ? RuntimeData->ClusterName : "local"}}, "ProcessInit")); - TaskRunnerPrepared = true; - - try { - Stat.AddCounters2(ev->Get()->Sensors); - - const auto& secureParams = ev->Get()->SecureParams; - const auto& taskParams = ev->Get()->TaskParams; - const auto& typeEnv = ev->Get()->TypeEnv; - const auto& holderFactory = ev->Get()->HolderFactory; - - Stat.Measure<void>("PrepareChannels", [&](){ - auto& inputs = Task.GetInputs(); - for (auto inputId = 0; inputId < inputs.size(); inputId++) { - auto& input = inputs[inputId]; - if (input.HasSource()) { - auto& source = SourcesMap[inputId]; - source.TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); - std::tie(source.SourceActor, source.Actor) = - SourceActorFactory->CreateDqSourceActor( - IDqSourceActorFactory::TArguments{ - .InputDesc = input, - .InputIndex = static_cast<ui64>(inputId), - .TxId = TraceId, - .SecureParams = secureParams, - .TaskParams = taskParams, - .Callback = this, - .TypeEnv = typeEnv, - .HolderFactory = holderFactory - }); - RegisterLocalChild(source.Actor); - } else { - for (auto& channel : input.GetChannels()) { - TInputChannel inChannel; - inChannel.InputId = inputId; - inChannel.ChannelId = channel.GetId(); - inChannel.ActorID = ResolveEndpoint(channel.GetSrcEndpoint()); - InputMap.emplace(inChannel.ActorID, inChannel); - InputChannelCount++; - } - } - } - - auto& outputs = Task.GetOutputs(); - for (auto outputId = 0; outputId < outputs.size(); outputId++) { - auto& output = outputs[outputId]; - if (output.HasSink()) { - auto& sink = SinksMap[outputId]; - sink.TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); - std::tie(sink.SinkActor, sink.Actor) = SinkActorFactory->CreateDqSinkActor( - IDqSinkActorFactory::TArguments { - .OutputDesc = output, - .OutputIndex = static_cast<ui64>(outputId), - .TxId = TraceId, - .SecureParams = secureParams, - .Callback = this, - .TypeEnv = typeEnv, - .HolderFactory = holderFactory - }); - RegisterLocalChild(sink.Actor); - } else { - for (auto& channel : output.GetChannels()) { - TOutputChannel outChannel; - outChannel.OutputId = outputId; - outChannel.ChannelId = channel.GetId(); - outChannel.ActorID = ResolveEndpoint(channel.GetDstEndpoint()); - OutputMap.emplace(outChannel.ActorID, outChannel); - OutChannelId2ActorId.emplace(outChannel.ChannelId, outChannel.ActorID); - } - } - } - }); - - if (PingTimeout) { - Schedule(PingPeriod, new TEvents::TEvWakeup); - } - } catch (...) { - OnError(); + Executer = ev->Sender; + Task = ev->Get()->Record.GetTask(); + + Yql::DqsProto::TTaskMeta taskMeta; + Task.GetMeta().UnpackTo(&taskMeta); + for (const auto& w : taskMeta.GetWorkerInfo()) { + AllWorkers.push_back(w); + } + StageId = taskMeta.GetStageId(); + Settings->Dispatch(taskMeta.GetSettings()); + Settings->FreezeDefaults(); + PullRequestTimeout = TDuration::MilliSeconds(Settings->PullRequestTimeoutMs.Get().GetOrElse(0)); + PingTimeout = TDuration::MilliSeconds(Settings->PingTimeoutMs.Get().GetOrElse(0)); + if (PingTimeout) { + PingPeriod = Max(PingTimeout/4, TDuration::MilliSeconds(1000)); + } + if (Settings->_FallbackOnRuntimeErrors.Get()) { + TString parts = Settings->_FallbackOnRuntimeErrors.Get().GetOrElse(""); + for (const auto& it : StringSplitter(parts).Split(',')) { + FallbackOn.insert(TString(it.Token())); + } + } + + NActors::IActor* actor; + std::tie(Actor, actor) = TaskRunnerActorFactory->Create(this, TraceId); + TaskRunnerActor = RegisterLocalChild(actor); + TDqTaskRunnerMemoryLimits limits; // used for local mode only + limits.ChannelBufferSize = 20_MB; + limits.OutputChunkMaxSize = 2_MB; + Send(TaskRunnerActor, new TEvTaskRunnerCreate(std::move(ev->Get()->Record.GetTask()), limits)); + } + + void OnTaskRunnerCreated(TEvTaskRunnerCreateFinished::TPtr& ev, const TActorContext& ) { + Stat.FlushCounter(Stat.GetCounterName("Actor", {{"ClusterName", RuntimeData ? RuntimeData->ClusterName : "local"}}, "ProcessInit")); + TaskRunnerPrepared = true; + + try { + Stat.AddCounters2(ev->Get()->Sensors); + + const auto& secureParams = ev->Get()->SecureParams; + const auto& taskParams = ev->Get()->TaskParams; + const auto& typeEnv = ev->Get()->TypeEnv; + const auto& holderFactory = ev->Get()->HolderFactory; + + Stat.Measure<void>("PrepareChannels", [&](){ + auto& inputs = Task.GetInputs(); + for (auto inputId = 0; inputId < inputs.size(); inputId++) { + auto& input = inputs[inputId]; + if (input.HasSource()) { + auto& source = SourcesMap[inputId]; + source.TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); + std::tie(source.SourceActor, source.Actor) = + SourceActorFactory->CreateDqSourceActor( + IDqSourceActorFactory::TArguments{ + .InputDesc = input, + .InputIndex = static_cast<ui64>(inputId), + .TxId = TraceId, + .SecureParams = secureParams, + .TaskParams = taskParams, + .Callback = this, + .TypeEnv = typeEnv, + .HolderFactory = holderFactory + }); + RegisterLocalChild(source.Actor); + } else { + for (auto& channel : input.GetChannels()) { + TInputChannel inChannel; + inChannel.InputId = inputId; + inChannel.ChannelId = channel.GetId(); + inChannel.ActorID = ResolveEndpoint(channel.GetSrcEndpoint()); + InputMap.emplace(inChannel.ActorID, inChannel); + InputChannelCount++; + } + } + } + + auto& outputs = Task.GetOutputs(); + for (auto outputId = 0; outputId < outputs.size(); outputId++) { + auto& output = outputs[outputId]; + if (output.HasSink()) { + auto& sink = SinksMap[outputId]; + sink.TypeEnv = const_cast<NKikimr::NMiniKQL::TTypeEnvironment*>(&typeEnv); + std::tie(sink.SinkActor, sink.Actor) = SinkActorFactory->CreateDqSinkActor( + IDqSinkActorFactory::TArguments { + .OutputDesc = output, + .OutputIndex = static_cast<ui64>(outputId), + .TxId = TraceId, + .SecureParams = secureParams, + .Callback = this, + .TypeEnv = typeEnv, + .HolderFactory = holderFactory + }); + RegisterLocalChild(sink.Actor); + } else { + for (auto& channel : output.GetChannels()) { + TOutputChannel outChannel; + outChannel.OutputId = outputId; + outChannel.ChannelId = channel.GetId(); + outChannel.ActorID = ResolveEndpoint(channel.GetDstEndpoint()); + OutputMap.emplace(outChannel.ActorID, outChannel); + OutChannelId2ActorId.emplace(outChannel.ChannelId, outChannel.ActorID); + } + } + } + }); + + if (PingTimeout) { + Schedule(PingPeriod, new TEvents::TEvWakeup); + } + } catch (...) { + OnError(); + } + } + + void OnPullRequest(TEvPullDataRequest::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(TRACE) << "TDqWorker::OnPullRequest " << ev->Sender; + + if (!TaskRunnerActor || !TaskRunnerPrepared) { + // waiting for initialization + TPullResponse response; + response.SetResponseType(EPullResponseType::YIELD); + Send(ev->Sender, MakeHolder<TEvPullDataResponse>(response)); + return; + } + + auto now = TInstant::Now(); + auto& outChannel = OutputMap[ev->Sender]; + outChannel.RequestTime = now; + + Send(TaskRunnerActor, new TEvPop(outChannel.ChannelId)); + } + + void OnChannelPopFinished(TEvChannelPopFinished::TPtr& ev, const NActors::TActorContext& ctx) { + try { + auto outputActorId = OutChannelId2ActorId[ev->Get()->ChannelId]; + auto& outChannel = OutputMap[outputActorId]; + TPullResponse response; + auto hasData = !ev->Get()->Data.empty(); + Stat.AddCounters2(ev->Get()->Sensors); + + if (hasData) { + response.SetResponseType(EPullResponseType::CONTINUE); + } else if (ev->Get()->Finished) { + outChannel.Finished = true; + response.SetResponseType(EPullResponseType::FINISH); + } else { + response.SetResponseType(EPullResponseType::YIELD); + } + + Y_VERIFY(ev->Get()->Data.size() <= 1); + + if (ev->Get()->Data.size() == 1) { + response.MutableData()->Swap(&ev->Get()->Data.front()); + } + + Stat.FlushCounters(response); + + Send(outputActorId, MakeHolder<TEvPullDataResponse>(response)); + + Run(ctx); + } catch (...) { + OnError(); } } - void OnPullRequest(TEvPullDataRequest::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(TRACE) << "TDqWorker::OnPullRequest " << ev->Sender; - - if (!TaskRunnerActor || !TaskRunnerPrepared) { - // waiting for initialization - TPullResponse response; - response.SetResponseType(EPullResponseType::YIELD); - Send(ev->Sender, MakeHolder<TEvPullDataResponse>(response)); - return; - } - - auto now = TInstant::Now(); - auto& outChannel = OutputMap[ev->Sender]; - outChannel.RequestTime = now; - - Send(TaskRunnerActor, new TEvPop(outChannel.ChannelId)); - } - - void OnChannelPopFinished(TEvChannelPopFinished::TPtr& ev, const NActors::TActorContext& ctx) { - try { - auto outputActorId = OutChannelId2ActorId[ev->Get()->ChannelId]; - auto& outChannel = OutputMap[outputActorId]; - TPullResponse response; - auto hasData = !ev->Get()->Data.empty(); - Stat.AddCounters2(ev->Get()->Sensors); - - if (hasData) { - response.SetResponseType(EPullResponseType::CONTINUE); - } else if (ev->Get()->Finished) { - outChannel.Finished = true; - response.SetResponseType(EPullResponseType::FINISH); - } else { - response.SetResponseType(EPullResponseType::YIELD); - } - - Y_VERIFY(ev->Get()->Data.size() <= 1); - - if (ev->Get()->Data.size() == 1) { - response.MutableData()->Swap(&ev->Get()->Data.front()); - } - - Stat.FlushCounters(response); - - Send(outputActorId, MakeHolder<TEvPullDataResponse>(response)); - - Run(ctx); - } catch (...) { - OnError(); + void OnPullResponse(TEvPullDataResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(TRACE) << "TDqWorker::OnPullResponse"; + + Stat.AddCounters(ev->Get()->Record); + + auto& channel = InputMap[ev->Sender]; + channel.Requested = false; + channel.Retries = 0; + channel.ResponseTime = TInstant::Now(); + auto responseType = ev->Get()->Record.GetResponseType(); + if (responseType == FINISH) { + channel.Finished = true; + FinishedChannels++; + } + if (responseType == YIELD) { + return; + } + if (responseType == ERROR) { + Send(SelfId(), new TEvError(ev->Get()->Record.GetErrorMessage(), {})); + return; + } + Y_VERIFY (responseType == FINISH || responseType == CONTINUE); + if (responseType == FINISH) { + Send(TaskRunnerActor, new TEvPush(channel.ChannelId)); + } else { + Send(TaskRunnerActor, new TEvPush( + channel.ChannelId, + std::move(*ev->Get()->Record.MutableData()))); } } - void OnPullResponse(TEvPullDataResponse::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(TRACE) << "TDqWorker::OnPullResponse"; - - Stat.AddCounters(ev->Get()->Record); - - auto& channel = InputMap[ev->Sender]; - channel.Requested = false; - channel.Retries = 0; - channel.ResponseTime = TInstant::Now(); - auto responseType = ev->Get()->Record.GetResponseType(); - if (responseType == FINISH) { - channel.Finished = true; - FinishedChannels++; - } - if (responseType == YIELD) { - return; - } - if (responseType == ERROR) { - Send(SelfId(), new TEvError(ev->Get()->Record.GetErrorMessage(), {})); - return; - } - Y_VERIFY (responseType == FINISH || responseType == CONTINUE); - if (responseType == FINISH) { - Send(TaskRunnerActor, new TEvPush(channel.ChannelId)); - } else { - Send(TaskRunnerActor, new TEvPush( - channel.ChannelId, - std::move(*ev->Get()->Record.MutableData()))); - } + void OnPingRequest(TEvPingRequest::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + Send(ev->Sender, MakeHolder<TEvPingResponse>(), IEventHandle::FlagTrackDelivery); } - void OnPingRequest(TEvPingRequest::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - Send(ev->Sender, MakeHolder<TEvPingResponse>(), IEventHandle::FlagTrackDelivery); - } - - void OnPingResponse(TEvPingResponse::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - auto& channel = InputMap[ev->Sender]; - channel.PingRequested = false; - } - - void OnWakeup() { - auto now = TInstant::Now(); - for (auto& [sender, channel] : InputMap) { - if (channel.Finished) { - continue; - } - if (!channel.PingRequested) { - Send(channel.ActorID, MakeHolder<TEvPingRequest>(), IEventHandle::FlagTrackDelivery); - channel.PingRequested = true; - channel.PingStartTime = now; - } else if ((now - channel.PingStartTime) > PingTimeout) { - Stat.AddCounter("PingTimeout", static_cast<ui64>(1)); - OnError("PingTimeout " + TimeoutInfo(channel.ActorID, now, channel.PingStartTime), true, true); - } - } - - Schedule(PingPeriod, new TEvents::TEvWakeup); - } - - NActors::TActorId ResolveEndpoint(const TEndpoint& ep) { + void OnPingResponse(TEvPingResponse::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto& channel = InputMap[ev->Sender]; + channel.PingRequested = false; + } + + void OnWakeup() { + auto now = TInstant::Now(); + for (auto& [sender, channel] : InputMap) { + if (channel.Finished) { + continue; + } + if (!channel.PingRequested) { + Send(channel.ActorID, MakeHolder<TEvPingRequest>(), IEventHandle::FlagTrackDelivery); + channel.PingRequested = true; + channel.PingStartTime = now; + } else if ((now - channel.PingStartTime) > PingTimeout) { + Stat.AddCounter("PingTimeout", static_cast<ui64>(1)); + OnError("PingTimeout " + TimeoutInfo(channel.ActorID, now, channel.PingStartTime), true, true); + } + } + + Schedule(PingPeriod, new TEvents::TEvWakeup); + } + + NActors::TActorId ResolveEndpoint(const TEndpoint& ep) { switch (ep.GetEndpointTypeCase()) { case TEndpoint::kActorId: return NActors::ActorIdFromProto(ep.GetActorId()); - case TEndpoint::kUri: - Y_ENSURE(false, "kUri is not supported"); + case TEndpoint::kUri: + Y_ENSURE(false, "kUri is not supported"); case TEndpoint::kTabletId: Y_ENSURE(false, "Tablets not supported by dqs"); case TEndpoint::ENDPOINTTYPE_NOT_SET: { @@ -535,314 +535,314 @@ private: } } - void OnUndelivered(TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext&) { - Stat.AddCounter("Undelivered", TDuration::MilliSeconds(1)); - - bool sendError = false; - auto maybeChannel = InputMap.find(ev->Sender); - if (ev->Get()->Reason != TEvents::TEvUndelivered::Disconnected) { - sendError = true; - } else if (ev->Sender.NodeId() == SelfId().NodeId()) { - sendError = true; - } else if (maybeChannel == InputMap.end()) { - sendError = true; - } else if (maybeChannel->second.Retries > Settings->MaxNetworkRetries.Get().GetOrElse(5)) { - sendError = true; - } - - if (sendError) { - TString message = "Undelivered from " + ToString(ev->Sender) + " to " + ToString(SelfId()) - + " reason: " + ToString(ev->Get()->Reason) + " retries: " + ToString( - maybeChannel == InputMap.end() - ? 0 - : maybeChannel->second.Retries - ) + " " + JobDebugInfo(ev->Sender); - OnError(message, /*retriable = */ true, /*fallback =*/ true); - } else if (ev->Get()->SourceType == TEvPullDataRequest::EventType) { - TActivationContext::Schedule(TDuration::MilliSeconds(100), - new IEventHandle(maybeChannel->second.ActorID, SelfId(), new TEvPullDataRequest(INPUT_SIZE), IEventHandle::FlagTrackDelivery) - ); - maybeChannel->second.Retries ++; - maybeChannel->second.Requested = true; - maybeChannel->second.RequestTime = TInstant::Now(); - } else { - // Ping - TActivationContext::Schedule(TDuration::MilliSeconds(100), - new IEventHandle(maybeChannel->second.ActorID, SelfId(), new TEvPingRequest(), IEventHandle::FlagTrackDelivery) - ); - maybeChannel->second.PingRequested = true; - } - } - - void Run(const TActorContext& ctx) { - Y_UNUSED(ctx); - if (TaskFinished) { - return; - } - - THashSet<ui32> inputChannels; - for (auto& input : InputMap) { - auto& channel = input.second; - if (!channel.Requested && !channel.Finished) { - inputChannels.insert(channel.ChannelId); - } - } - - Send(TaskRunnerActor, new TEvContinueRun(std::move(inputChannels), Settings->MemoryLimit.Get().GetOrElse(0))); - } - - void OnRunFinished(TEvTaskRunFinished::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ctx); - auto res = ev->Get()->RunStatus; - if (RuntimeData) { - ::TRusage delta; - delta.Stime = TDuration::MicroSeconds(ev->Get()->Rusage.Stime); - delta.Utime = TDuration::MicroSeconds(ev->Get()->Rusage.Utime); - delta.MajorPageFaults = ev->Get()->Rusage.MajorPageFaults; - RuntimeData->AddRusageDelta(delta); - } - - Stat.AddCounters2(ev->Get()->Sensors); - + void OnUndelivered(TEvents::TEvUndelivered::TPtr& ev, const NActors::TActorContext&) { + Stat.AddCounter("Undelivered", TDuration::MilliSeconds(1)); + + bool sendError = false; + auto maybeChannel = InputMap.find(ev->Sender); + if (ev->Get()->Reason != TEvents::TEvUndelivered::Disconnected) { + sendError = true; + } else if (ev->Sender.NodeId() == SelfId().NodeId()) { + sendError = true; + } else if (maybeChannel == InputMap.end()) { + sendError = true; + } else if (maybeChannel->second.Retries > Settings->MaxNetworkRetries.Get().GetOrElse(5)) { + sendError = true; + } + + if (sendError) { + TString message = "Undelivered from " + ToString(ev->Sender) + " to " + ToString(SelfId()) + + " reason: " + ToString(ev->Get()->Reason) + " retries: " + ToString( + maybeChannel == InputMap.end() + ? 0 + : maybeChannel->second.Retries + ) + " " + JobDebugInfo(ev->Sender); + OnError(message, /*retriable = */ true, /*fallback =*/ true); + } else if (ev->Get()->SourceType == TEvPullDataRequest::EventType) { + TActivationContext::Schedule(TDuration::MilliSeconds(100), + new IEventHandle(maybeChannel->second.ActorID, SelfId(), new TEvPullDataRequest(INPUT_SIZE), IEventHandle::FlagTrackDelivery) + ); + maybeChannel->second.Retries ++; + maybeChannel->second.Requested = true; + maybeChannel->second.RequestTime = TInstant::Now(); + } else { + // Ping + TActivationContext::Schedule(TDuration::MilliSeconds(100), + new IEventHandle(maybeChannel->second.ActorID, SelfId(), new TEvPingRequest(), IEventHandle::FlagTrackDelivery) + ); + maybeChannel->second.PingRequested = true; + } + } + + void Run(const TActorContext& ctx) { + Y_UNUSED(ctx); + if (TaskFinished) { + return; + } + + THashSet<ui32> inputChannels; + for (auto& input : InputMap) { + auto& channel = input.second; + if (!channel.Requested && !channel.Finished) { + inputChannels.insert(channel.ChannelId); + } + } + + Send(TaskRunnerActor, new TEvContinueRun(std::move(inputChannels), Settings->MemoryLimit.Get().GetOrElse(0))); + } + + void OnRunFinished(TEvTaskRunFinished::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + auto res = ev->Get()->RunStatus; + if (RuntimeData) { + ::TRusage delta; + delta.Stime = TDuration::MicroSeconds(ev->Get()->Rusage.Stime); + delta.Utime = TDuration::MicroSeconds(ev->Get()->Rusage.Utime); + delta.MajorPageFaults = ev->Get()->Rusage.MajorPageFaults; + RuntimeData->AddRusageDelta(delta); + } + + Stat.AddCounters2(ev->Get()->Sensors); + switch (res) { - case ERunStatus::Finished: { - TaskFinished = true; + case ERunStatus::Finished: { + TaskFinished = true; break; - } - case ERunStatus::PendingInput: { - auto now = TInstant::Now(); - for (auto& [sender, channel] : InputMap) { - if (!channel.Requested && !channel.Finished) { - auto freeSpace = ev->Get()->InputChannelFreeSpace.find(channel.ChannelId); - auto hasFreeSpace = freeSpace == ev->Get()->InputChannelFreeSpace.end() - || freeSpace->second > 0; - if (hasFreeSpace) { - YQL_LOG(TRACE) << "Send TEvPullDataRequest to " << - channel.ActorID << " from " << - SelfId(); - Send(channel.ActorID, MakeHolder<TEvPullDataRequest>(INPUT_SIZE), IEventHandle::FlagTrackDelivery); - channel.Requested = true; - channel.RequestTime = now; - } - } else if (channel.Requested && !channel.Finished) { - if (PullRequestTimeout && (now - channel.RequestTime) > PullRequestTimeout) { - Stat.AddCounter("ReadTimeout", static_cast<ui64>(1)); - OnError("PullTimeout " + TimeoutInfo(channel.ActorID, now, channel.RequestTime), false, true); - } + } + case ERunStatus::PendingInput: { + auto now = TInstant::Now(); + for (auto& [sender, channel] : InputMap) { + if (!channel.Requested && !channel.Finished) { + auto freeSpace = ev->Get()->InputChannelFreeSpace.find(channel.ChannelId); + auto hasFreeSpace = freeSpace == ev->Get()->InputChannelFreeSpace.end() + || freeSpace->second > 0; + if (hasFreeSpace) { + YQL_LOG(TRACE) << "Send TEvPullDataRequest to " << + channel.ActorID << " from " << + SelfId(); + Send(channel.ActorID, MakeHolder<TEvPullDataRequest>(INPUT_SIZE), IEventHandle::FlagTrackDelivery); + channel.Requested = true; + channel.RequestTime = now; + } + } else if (channel.Requested && !channel.Finished) { + if (PullRequestTimeout && (now - channel.RequestTime) > PullRequestTimeout) { + Stat.AddCounter("ReadTimeout", static_cast<ui64>(1)); + OnError("PullTimeout " + TimeoutInfo(channel.ActorID, now, channel.RequestTime), false, true); + } } } - - for (auto& [inputIndex, source] : SourcesMap) { - auto& freeSpace = source.FreeSpace; - auto it = ev->Get()->SourcesFreeSpace.find(inputIndex); - if (it != ev->Get()->SourcesFreeSpace.end()) { - freeSpace = it->second; - } - if (freeSpace < 0 || source.PushStarted || source.Finished) { - continue; - } - auto guard = source.TypeEnv->BindAllocator(); - NKikimr::NMiniKQL::TUnboxedValueVector batch; - bool finished = false; - const i64 space = source.SourceActor->GetSourceData(batch, finished, freeSpace); - const ui64 index = inputIndex; - if (space <= 0) { - continue; - } - source.PushStarted = true; - source.Finished = finished; - - Actor->SourcePush(0, index, std::move(batch), space, finished); - } + + for (auto& [inputIndex, source] : SourcesMap) { + auto& freeSpace = source.FreeSpace; + auto it = ev->Get()->SourcesFreeSpace.find(inputIndex); + if (it != ev->Get()->SourcesFreeSpace.end()) { + freeSpace = it->second; + } + if (freeSpace < 0 || source.PushStarted || source.Finished) { + continue; + } + auto guard = source.TypeEnv->BindAllocator(); + NKikimr::NMiniKQL::TUnboxedValueVector batch; + bool finished = false; + const i64 space = source.SourceActor->GetSourceData(batch, finished, freeSpace); + const ui64 index = inputIndex; + if (space <= 0) { + continue; + } + source.PushStarted = true; + source.Finished = finished; + + Actor->SourcePush(0, index, std::move(batch), space, finished); + } break; - } - case ERunStatus::PendingOutput: { - for (auto& [index, sink] : SinksMap) { - const i64 sinkActorFreeSpaceBeforeSend = sink.SinkActor->GetFreeSpace(); - if (sinkActorFreeSpaceBeforeSend > 0 && !sink.Finished) { - Send(TaskRunnerActor, new TEvSinkPop(index, sinkActorFreeSpaceBeforeSend)); - } - } + } + case ERunStatus::PendingOutput: { + for (auto& [index, sink] : SinksMap) { + const i64 sinkActorFreeSpaceBeforeSend = sink.SinkActor->GetFreeSpace(); + if (sinkActorFreeSpaceBeforeSend > 0 && !sink.Finished) { + Send(TaskRunnerActor, new TEvSinkPop(index, sinkActorFreeSpaceBeforeSend)); + } + } break; - } + } + } + } + + TString TimeoutInfo(TActorId actorID, TInstant now, TInstant startTime) { + TString message = ToString(actorID) + + " to " + ToString(SelfId()) + + " duration " + ToString(now - startTime) + + " stageId " + ToString(StageId) + " " + + JobDebugInfo(actorID); + + return message; + } + + TString JobDebugInfo(TActorId actorID) { + TString message; + TString from, to; + for (const auto& w : AllWorkers) { + if (w.GetNodeId() == actorID.NodeId() && w.GetJobId() && w.GetOperationId()) { + from = "J/O=" + w.GetJobId() + "/" + w.GetOperationId(); + } + if (w.GetNodeId() == SelfId().NodeId() && w.GetJobId() && w.GetOperationId()) { + to = "J/O=" + w.GetJobId() + "/" + w.GetOperationId(); + } + } + + if (from && to) { + message += "(" + from + "->" + to + " )"; } + return message; + } + + void Dump() { + auto now = TInstant::Now(); + for (const auto& [actorId, channel] : InputMap) { + if (!channel.Finished) { + if (channel.Requested) { + YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << (now - channel.RequestTime) << " Requested? " << channel.Requested; + if (RuntimeData) { + RuntimeData->UpdateChannelInputDelay(now - channel.RequestTime); + } + } else { + YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << (now - channel.ResponseTime) << " Requested? " << channel.Requested; + if (RuntimeData) { + RuntimeData->UpdateChannelInputDelay(now - channel.ResponseTime); + } + } + } else { + YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << " Finished"; + if (RuntimeData) { + RuntimeData->UpdateChannelInputDelay(TDuration::Seconds(0)); + } + } + } + + for (const auto& [actorId, channel] : OutputMap) { + if (!channel.Finished) { + YQL_LOG(DEBUG) << "Output " << JobDebugInfo(actorId) << (now - channel.RequestTime); + if (RuntimeData) { + RuntimeData->UpdateChannelOutputDelay(now - channel.RequestTime); + } + } else { + YQL_LOG(DEBUG) << "Output " << JobDebugInfo(actorId) << " Finished"; + if (RuntimeData) { + RuntimeData->UpdateChannelOutputDelay(TDuration::Seconds(0)); + } + } + } + } + + /*____________________ SourceActorEvents __________________*/ + void OnNewSourceDataArrived(ui64 inputIndex) override { + try { + if (!TaskRunnerPrepared) { + return; + } + auto& source = SourcesMap[inputIndex]; + source.HasData = true; + Send(SelfId(), new TEvContinueRun()); + } catch (...) { + OnError(); + } + } + void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override { + Y_UNUSED(inputIndex); + OnError(issues.ToString(), !isFatal, !isFatal); } - - TString TimeoutInfo(TActorId actorID, TInstant now, TInstant startTime) { - TString message = ToString(actorID) - + " to " + ToString(SelfId()) - + " duration " + ToString(now - startTime) - + " stageId " + ToString(StageId) + " " - + JobDebugInfo(actorID); - - return message; - } - - TString JobDebugInfo(TActorId actorID) { - TString message; - TString from, to; - for (const auto& w : AllWorkers) { - if (w.GetNodeId() == actorID.NodeId() && w.GetJobId() && w.GetOperationId()) { - from = "J/O=" + w.GetJobId() + "/" + w.GetOperationId(); - } - if (w.GetNodeId() == SelfId().NodeId() && w.GetJobId() && w.GetOperationId()) { - to = "J/O=" + w.GetJobId() + "/" + w.GetOperationId(); - } - } - - if (from && to) { - message += "(" + from + "->" + to + " )"; - } - return message; - } - - void Dump() { - auto now = TInstant::Now(); - for (const auto& [actorId, channel] : InputMap) { - if (!channel.Finished) { - if (channel.Requested) { - YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << (now - channel.RequestTime) << " Requested? " << channel.Requested; - if (RuntimeData) { - RuntimeData->UpdateChannelInputDelay(now - channel.RequestTime); - } - } else { - YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << (now - channel.ResponseTime) << " Requested? " << channel.Requested; - if (RuntimeData) { - RuntimeData->UpdateChannelInputDelay(now - channel.ResponseTime); - } - } - } else { - YQL_LOG(DEBUG) << "Input " << JobDebugInfo(actorId) << " Finished"; - if (RuntimeData) { - RuntimeData->UpdateChannelInputDelay(TDuration::Seconds(0)); - } - } - } - - for (const auto& [actorId, channel] : OutputMap) { - if (!channel.Finished) { - YQL_LOG(DEBUG) << "Output " << JobDebugInfo(actorId) << (now - channel.RequestTime); - if (RuntimeData) { - RuntimeData->UpdateChannelOutputDelay(now - channel.RequestTime); - } - } else { - YQL_LOG(DEBUG) << "Output " << JobDebugInfo(actorId) << " Finished"; - if (RuntimeData) { - RuntimeData->UpdateChannelOutputDelay(TDuration::Seconds(0)); - } - } - } - } - - /*____________________ SourceActorEvents __________________*/ - void OnNewSourceDataArrived(ui64 inputIndex) override { - try { - if (!TaskRunnerPrepared) { - return; - } - auto& source = SourcesMap[inputIndex]; - source.HasData = true; - Send(SelfId(), new TEvContinueRun()); - } catch (...) { - OnError(); - } - } - void OnSourceError(ui64 inputIndex, const TIssues& issues, bool isFatal) override { - Y_UNUSED(inputIndex); - OnError(issues.ToString(), !isFatal, !isFatal); - } - void OnSourcePushFinished(TEvSourcePushFinished::TPtr& ev, const TActorContext& ctx) { - auto index = ev->Get()->Index; - auto& source = SourcesMap[index]; - source.PushStarted = false; - Run(ctx); - } - /*_________________________________________________________*/ - /*______________________ SinkActorEvents __________________*/ - void ResumeExecution() override { - Send(SelfId(), new TEvContinueRun()); - } - - void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override { - Y_UNUSED(outputIndex); - OnError(issues.ToString(), !isFatal, !isFatal); - } - - void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) override { - Y_UNUSED(state); - Y_UNUSED(outputIndex); - Y_UNUSED(checkpoint); - OnError("Unimplemented"); - } - - void SinkSend( - ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - TMaybe<NDqProto::TCheckpoint>&& checkpoint, - i64 size, - i64 checkpointSize, - bool finished, - bool changed) override - { - Y_UNUSED(checkpointSize); Y_UNUSED(checkpoint); Y_UNUSED(changed); - auto& sink = SinksMap[index]; - sink.Finished = finished; - sink.SinkActor->SendData(std::move(batch), size, {}, finished); - } - - /*_________________________________________________________*/ - - IDqSourceActorFactory::TPtr SourceActorFactory; - IDqSinkActorFactory::TPtr SinkActorFactory; - ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; - NTaskRunnerActor::ITaskRunnerActor* Actor = nullptr; - TActorId TaskRunnerActor; - - NDqProto::TDqTask Task; + void OnSourcePushFinished(TEvSourcePushFinished::TPtr& ev, const TActorContext& ctx) { + auto index = ev->Get()->Index; + auto& source = SourcesMap[index]; + source.PushStarted = false; + Run(ctx); + } + /*_________________________________________________________*/ + /*______________________ SinkActorEvents __________________*/ + void ResumeExecution() override { + Send(SelfId(), new TEvContinueRun()); + } + + void OnSinkError(ui64 outputIndex, const TIssues& issues, bool isFatal) override { + Y_UNUSED(outputIndex); + OnError(issues.ToString(), !isFatal, !isFatal); + } + + void OnSinkStateSaved(NDqProto::TSinkState&& state, ui64 outputIndex, const NDqProto::TCheckpoint& checkpoint) override { + Y_UNUSED(state); + Y_UNUSED(outputIndex); + Y_UNUSED(checkpoint); + OnError("Unimplemented"); + } + + void SinkSend( + ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + TMaybe<NDqProto::TCheckpoint>&& checkpoint, + i64 size, + i64 checkpointSize, + bool finished, + bool changed) override + { + Y_UNUSED(checkpointSize); Y_UNUSED(checkpoint); Y_UNUSED(changed); + auto& sink = SinksMap[index]; + sink.Finished = finished; + sink.SinkActor->SendData(std::move(batch), size, {}, finished); + } + + /*_________________________________________________________*/ + + IDqSourceActorFactory::TPtr SourceActorFactory; + IDqSinkActorFactory::TPtr SinkActorFactory; + ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; + NTaskRunnerActor::ITaskRunnerActor* Actor = nullptr; + TActorId TaskRunnerActor; + + NDqProto::TDqTask Task; ui64 StageId = 0; - bool TaskRunnerPrepared = false; - - THashMap<NActors::TActorId, TInputChannel> InputMap; - THashMap<NActors::TActorId, TOutputChannel> OutputMap; - THashMap<ui64, NActors::TActorId> OutChannelId2ActorId; - THashMap<ui64, TSourceInfo> SourcesMap; - THashMap<ui64, TSinkInfo> SinksMap; - - ui32 InputChannelCount = 0; - ui32 FinishedChannels = 0; - - NActors::TActorId Executer; - - TWorkerRuntimeData* RuntimeData; - bool TaskFinished = false; - - const TString TraceId; - - TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); - TDuration PullRequestTimeout; - TDuration PingTimeout; - TDuration PingPeriod; - NYql::TCounters Stat; - - TVector<Yql::DqsProto::TWorkerInfo> AllWorkers; - THashSet<TString> FallbackOn; -}; - -NActors::IActor* CreateWorkerActor( - TWorkerRuntimeData* runtimeData, - const TString& traceId, - const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, - const IDqSourceActorFactory::TPtr& sourceActorFactory, - const IDqSinkActorFactory::TPtr& sinkActorFactory) -{ - Y_VERIFY(taskRunnerActorFactory); - return new TLogWrapReceive( - new TDqWorker( - taskRunnerActorFactory, - sourceActorFactory, - sinkActorFactory, - runtimeData, - traceId), traceId); + bool TaskRunnerPrepared = false; + + THashMap<NActors::TActorId, TInputChannel> InputMap; + THashMap<NActors::TActorId, TOutputChannel> OutputMap; + THashMap<ui64, NActors::TActorId> OutChannelId2ActorId; + THashMap<ui64, TSourceInfo> SourcesMap; + THashMap<ui64, TSinkInfo> SinksMap; + + ui32 InputChannelCount = 0; + ui32 FinishedChannels = 0; + + NActors::TActorId Executer; + + TWorkerRuntimeData* RuntimeData; + bool TaskFinished = false; + + const TString TraceId; + + TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); + TDuration PullRequestTimeout; + TDuration PingTimeout; + TDuration PingPeriod; + NYql::TCounters Stat; + + TVector<Yql::DqsProto::TWorkerInfo> AllWorkers; + THashSet<TString> FallbackOn; +}; + +NActors::IActor* CreateWorkerActor( + TWorkerRuntimeData* runtimeData, + const TString& traceId, + const ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, + const IDqSourceActorFactory::TPtr& sourceActorFactory, + const IDqSinkActorFactory::TPtr& sinkActorFactory) +{ + Y_VERIFY(taskRunnerActorFactory); + return new TLogWrapReceive( + new TDqWorker( + taskRunnerActorFactory, + sourceActorFactory, + sinkActorFactory, + runtimeData, + traceId), traceId); } - -} // namespace NYql::NDqs + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.h b/ydb/library/yql/providers/dq/actors/worker_actor.h index eeceb2e1e9..4d39687f5b 100644 --- a/ydb/library/yql/providers/dq/actors/worker_actor.h +++ b/ydb/library/yql/providers/dq/actors/worker_actor.h @@ -1,7 +1,7 @@ #pragma once #include "events.h" -#include "actor_helpers.h" +#include "actor_helpers.h" #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h> @@ -14,18 +14,18 @@ #include <library/cpp/actors/core/actor.h> #include <ydb/library/yql/providers/dq/counters/counters.h> - -namespace NYql { - struct TWorkerRuntimeData; -} - + +namespace NYql { + struct TWorkerRuntimeData; +} + namespace NYql::NDqs { - NActors::IActor* CreateWorkerActor( - TWorkerRuntimeData* runtimeData, - const TString& traceId, - const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, - const NDq::IDqSourceActorFactory::TPtr& sourceActorFactory, - const NDq::IDqSinkActorFactory::TPtr& sinkActorFactory); + NActors::IActor* CreateWorkerActor( + TWorkerRuntimeData* runtimeData, + const TString& traceId, + const NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr& taskRunnerActorFactory, + const NDq::IDqSourceActorFactory::TPtr& sourceActorFactory, + const NDq::IDqSinkActorFactory::TPtr& sinkActorFactory); -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/actors/ya.make b/ydb/library/yql/providers/dq/actors/ya.make index ca4a091e22..14e93b587d 100644 --- a/ydb/library/yql/providers/dq/actors/ya.make +++ b/ydb/library/yql/providers/dq/actors/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - +LIBRARY() + OWNER(g:yql) - -SRCS( + +SRCS( compute_actor.cpp - events.cpp - executer_actor.cpp - execution_helpers.cpp + events.cpp + executer_actor.cpp + execution_helpers.cpp graph_execution_events_actor.cpp - resource_allocator.cpp + resource_allocator.cpp task_controller.cpp worker_actor.cpp result_aggregator.cpp result_receiver.cpp full_result_writer.cpp proto_builder.cpp -) - -PEERDIR( - library/cpp/actors/core +) + +PEERDIR( + library/cpp/actors/core library/cpp/yson ydb/core/base ydb/library/mkql_proto @@ -46,8 +46,8 @@ PEERDIR( ydb/library/yql/providers/dq/task_runner_actor ydb/library/yql/providers/dq/worker_manager ydb/library/yql/providers/dq/worker_manager/interface -) - +) + YQL_LAST_ABI_VERSION() - -END() + +END() diff --git a/ydb/library/yql/providers/dq/api/grpc/api.proto b/ydb/library/yql/providers/dq/api/grpc/api.proto index ee4fd31eed..ef620dca4f 100644 --- a/ydb/library/yql/providers/dq/api/grpc/api.proto +++ b/ydb/library/yql/providers/dq/api/grpc/api.proto @@ -6,19 +6,19 @@ import "ydb/library/yql/providers/dq/api/protos/service.proto"; package Yql.DqsProto; service DqService { - rpc ExecuteGraph (ExecuteGraphRequest) returns (ExecuteGraphResponse); + rpc ExecuteGraph (ExecuteGraphRequest) returns (ExecuteGraphResponse); rpc SvnRevision (SvnRevisionRequest) returns (SvnRevisionResponse); - rpc RegisterNode (RegisterNodeRequest) returns (RegisterNodeResponse); - rpc OpenSession (OpenSessionRequest) returns (OpenSessionResponse); - rpc CloseSession (CloseSessionRequest) returns (CloseSessionResponse); - rpc PingSession (PingSessionRequest) returns (PingSessionResponse); - rpc ClusterStatus (ClusterStatusRequest) returns (ClusterStatusResponse); - rpc OperationStop (OperationStopRequest) returns (OperationStopResponse); + rpc RegisterNode (RegisterNodeRequest) returns (RegisterNodeResponse); + rpc OpenSession (OpenSessionRequest) returns (OpenSessionResponse); + rpc CloseSession (CloseSessionRequest) returns (CloseSessionResponse); + rpc PingSession (PingSessionRequest) returns (PingSessionResponse); + rpc ClusterStatus (ClusterStatusRequest) returns (ClusterStatusResponse); + rpc OperationStop (OperationStopRequest) returns (OperationStopResponse); rpc QueryStatus (QueryStatusRequest) returns (QueryStatusResponse); - rpc JobStop (JobStopRequest) returns (JobStopResponse); + rpc JobStop (JobStopRequest) returns (JobStopResponse); rpc GetMaster (GetMasterRequest) returns (GetMasterResponse); rpc ConfigureFailureInjector (ConfigureFailureInjectorRequest) returns (ConfigureFailureInjectorResponse); rpc IsReady (IsReadyRequest) returns (IsReadyResponse); - rpc Routes (RoutesRequest) returns (RoutesResponse); - rpc Benchmark (BenchmarkRequest) returns (BenchmarkResponse); + rpc Routes (RoutesRequest) returns (RoutesResponse); + rpc Benchmark (BenchmarkRequest) returns (BenchmarkResponse); } diff --git a/ydb/library/yql/providers/dq/api/grpc/ya.make b/ydb/library/yql/providers/dq/api/grpc/ya.make index 19b81256ee..4a67815a3d 100644 --- a/ydb/library/yql/providers/dq/api/grpc/ya.make +++ b/ydb/library/yql/providers/dq/api/grpc/ya.make @@ -3,7 +3,7 @@ PROTO_LIBRARY() GRPC() OWNER( - g:yql + g:yql ) SRCS( diff --git a/ydb/library/yql/providers/dq/api/protos/dqs.proto b/ydb/library/yql/providers/dq/api/protos/dqs.proto index 15fcd15941..09aecfcf92 100644 --- a/ydb/library/yql/providers/dq/api/protos/dqs.proto +++ b/ydb/library/yql/providers/dq/api/protos/dqs.proto @@ -13,102 +13,102 @@ import "library/cpp/actors/protos/actors.proto"; message TAllocateWorkersRequest { uint32 Count = 1; - // Yql.DqsProto.ExecuteQueryRequest Request = 2; // unused - - bool IsForwarded = 3; - uint64 ResourceId = 4; // debug resource allocation on worker nodes - - repeated Yql.DqsProto.TFile Files = 5; // deprecated - - string TraceId = 6; - - // repeated Yql.DqsProto.DatabaseDescription Databases = 7; // unused + // Yql.DqsProto.ExecuteQueryRequest Request = 2; // unused + + bool IsForwarded = 3; + uint64 ResourceId = 4; // debug resource allocation on worker nodes + + repeated Yql.DqsProto.TFile Files = 5; // deprecated + + string TraceId = 6; + + // repeated Yql.DqsProto.DatabaseDescription Databases = 7; // unused string User = 8; - repeated Yql.DqsProto.TWorkerFilter WorkerFilterPerTask = 9; - - uint32 WorkersCount = 10; - - bool CreateComputeActor = 11; // false - string ComputeActorType = 15; - repeated NYql.NDqProto.TDqTask Task = 12; // used for compute actor - NActorsProto.TActorId ResultActorId = 13; // used for compute actor + repeated Yql.DqsProto.TWorkerFilter WorkerFilterPerTask = 9; - uint64 FreeWorkerAfterMs = 14; + uint32 WorkersCount = 10; + + bool CreateComputeActor = 11; // false + string ComputeActorType = 15; + repeated NYql.NDqProto.TDqTask Task = 12; // used for compute actor + NActorsProto.TActorId ResultActorId = 13; // used for compute actor + + uint64 FreeWorkerAfterMs = 14; } message TWorkerGroup { - uint64 ResourceId = 1; + uint64 ResourceId = 1; repeated NActorsProto.TActorId WorkerActor = 2; - repeated Yql.DqsProto.TWorkerInfo Worker = 3; + repeated Yql.DqsProto.TWorkerInfo Worker = 3; +} + +enum EErrorCode { + EUNKNOWN = 0; + EMISMATCH = 1; + ETERMINATING = 2; + EFOLLOWER = 3; + EINITIALIZATION = 4; } -enum EErrorCode { - EUNKNOWN = 0; - EMISMATCH = 1; - ETERMINATING = 2; - EFOLLOWER = 3; - EINITIALIZATION = 4; -} - -message TDqError { - string Message = 1; - EErrorCode ErrorCode = 2; +message TDqError { + string Message = 1; + EErrorCode ErrorCode = 2; } message TAllocateWorkersResponse { oneof TResponse { - TWorkerGroup Workers = 1; // contains actorIds - TDqError Error = 2; - TWorkerGroup Nodes = 3; // contains nodeIds - }; - - repeated TMetric Metric = 4; + TWorkerGroup Workers = 1; // contains actorIds + TDqError Error = 2; + TWorkerGroup Nodes = 3; // contains nodeIds + }; + + repeated TMetric Metric = 4; } message TFreeWorkersNotify { - uint64 ResourceId = 1; - bool IsForwarded = 2; - - string TraceId = 6; - // repeated TWorker.TGuid FailedWorker = 7; // reserved - repeated string FailedWorkerGuid = 8; -} - -message TEvRegisterNode { - Yql.DqsProto.RegisterNodeRequest Request = 1; - bool IsForwarded = 4; -} - -message TEvRegisterNodeResponse { - Yql.DqsProto.RegisterNodeResponse Response = 1; -} - -message TEvJobStop { - Yql.DqsProto.JobStopRequest Request = 1; - bool IsForwarded = 2; -} - -message TEvJobStopResponse { -} - -message TEvClusterStatus { - bool IsForwarded = 4; -} - -message TEvClusterStatusResponse { - Yql.DqsProto.ClusterStatusResponse Response = 1; -} - -message TEvOperationStop { - Yql.DqsProto.OperationStopRequest Request = 1; - bool IsForwarded = 2; -} - -message TEvOperationStopResponse { -} - + uint64 ResourceId = 1; + bool IsForwarded = 2; + + string TraceId = 6; + // repeated TWorker.TGuid FailedWorker = 7; // reserved + repeated string FailedWorkerGuid = 8; +} + +message TEvRegisterNode { + Yql.DqsProto.RegisterNodeRequest Request = 1; + bool IsForwarded = 4; +} + +message TEvRegisterNodeResponse { + Yql.DqsProto.RegisterNodeResponse Response = 1; +} + +message TEvJobStop { + Yql.DqsProto.JobStopRequest Request = 1; + bool IsForwarded = 2; +} + +message TEvJobStopResponse { +} + +message TEvClusterStatus { + bool IsForwarded = 4; +} + +message TEvClusterStatusResponse { + Yql.DqsProto.ClusterStatusResponse Response = 1; +} + +message TEvOperationStop { + Yql.DqsProto.OperationStopRequest Request = 1; + bool IsForwarded = 2; +} + +message TEvOperationStopResponse { +} + message TEvQueryStatus { Yql.DqsProto.QueryStatusRequest Request = 1; bool IsForwarded = 2; @@ -127,13 +127,13 @@ message TEvIsReadyResponse { bool IsReady = 1; } -message TEvRoutesRequest { -} - -message TEvRoutesResponse { - Yql.DqsProto.RoutesResponse Response = 1; -} - +message TEvRoutesRequest { +} + +message TEvRoutesResponse { + Yql.DqsProto.RoutesResponse Response = 1; +} + message TPullRequest { uint32 RowThreshold = 1; } @@ -143,68 +143,68 @@ enum EPullResponseType { CONTINUE = 1; FINISH = 2; YIELD = 3; - ERROR = 4; -} - -message TMetric { - string Name = 1; - int64 Sum = 2; - int64 Max = 3; - int64 Min = 4; - int64 Avg = 5; - int64 Count = 6; -} - -message TRusage { - int64 Utime = 1; - int64 Stime = 2; - int64 MajorPageFaults = 3; -} - + ERROR = 4; +} + +message TMetric { + string Name = 1; + int64 Sum = 2; + int64 Max = 3; + int64 Min = 4; + int64 Avg = 5; + int64 Count = 6; +} + +message TRusage { + int64 Utime = 1; + int64 Stime = 2; + int64 MajorPageFaults = 3; +} + message TPullResponse { EPullResponseType ResponseType = 1; TData Data = 2; - repeated TMetric Metric = 3; - string ErrorMessage = 4; + repeated TMetric Metric = 3; + string ErrorMessage = 4; } message TQueryResponse { Ydb.ResultSet ResultSet = 1; repeated Ydb.Issue.IssueMessage Issues = 2; - bytes Yson = 3; - bool Retriable = 4; - bool NeedFallback = 6; - repeated TMetric Metric = 5; - bool Truncated = 7; + bytes Yson = 3; + bool Retriable = 4; + bool NeedFallback = 6; + repeated TMetric Metric = 5; + bool Truncated = 7; uint64 RowsCount = 8; } -message TDqFailure { +message TDqFailure { repeated Ydb.Issue.IssueMessage Issues = 4; - repeated TMetric Metric = 5; - bool Retriable = 6; - bool NeedFallback = 7; -}; - -message TGraphRequest { - Yql.DqsProto.ExecuteGraphRequest Request = 1; + repeated TMetric Metric = 5; + bool Retriable = 6; + bool NeedFallback = 7; +}; + +message TGraphRequest { + Yql.DqsProto.ExecuteGraphRequest Request = 1; NActorsProto.TActorId ControlId = 2; NActorsProto.TActorId ResultId = 3; NActorsProto.TActorId CheckPointCoordinatorId = 4; // may be empty -} - -message TDqTaskRequest { - NYql.NDqProto.TDqTask Task = 1; -} - +} + +message TDqTaskRequest { + NYql.NDqProto.TDqTask Task = 1; +} + message TReadyState { NActorsProto.TActorId SourceId = 1; string ResultType = 2; - repeated TMetric Metric = 3; - repeated NYql.NDqProto.TDqTask Task = 4; // used for compute actor - repeated NActorsProto.TActorId ActorId = 5; // used for compute actor + repeated TMetric Metric = 3; + repeated NYql.NDqProto.TDqTask Task = 4; // used for compute actor + repeated NActorsProto.TActorId ActorId = 5; // used for compute actor } - + enum EGraphExecutionEventType { SYNC = 0; START = 1; @@ -216,7 +216,7 @@ enum EGraphExecutionEventType { message TGraphExecutionEvent { EGraphExecutionEventType EventType = 1; google.protobuf.Any Message = 2; - optional string ErrorMessage = 3; + optional string ErrorMessage = 3; message TMap { map<string, bytes> Data = 1; @@ -240,10 +240,10 @@ message TFullResultWriterStatusResponse { optional string ErrorMessage = 2; } -message TDqTaskPrepareResult { - bool Result = 1; - repeated TMetric Metric = 2; -} +message TDqTaskPrepareResult { + bool Result = 1; + repeated TMetric Metric = 2; +} message TEvGetMasterRequest { bool IsForwarded = 1; @@ -260,11 +260,11 @@ message TEvConfigureFailureInjectorRequest { message TEvConfigureFailureInjectorRequestResponse { Yql.DqsProto.ConfigureFailureInjectorResponse Response = 1; } - -message TPingRequest { - -} - -message TPingResponse { - -} + +message TPingRequest { + +} + +message TPingResponse { + +} diff --git a/ydb/library/yql/providers/dq/api/protos/service.proto b/ydb/library/yql/providers/dq/api/protos/service.proto index 0e1e882473..b91351e5d1 100644 --- a/ydb/library/yql/providers/dq/api/protos/service.proto +++ b/ydb/library/yql/providers/dq/api/protos/service.proto @@ -7,117 +7,117 @@ import "ydb/library/yql/dq/proto/dq_tasks.proto"; package Yql.DqsProto; -message TAttr { - string Name = 1; - string Value = 2; -} - -message ResponseMetric { - string Name = 1; - int64 Sum = 2; - int64 Max = 3; - int64 Min = 4; - int64 Avg = 5; - int64 Count = 6; -} - +message TAttr { + string Name = 1; + string Value = 2; +} + +message ResponseMetric { + string Name = 1; + int64 Sum = 2; + int64 Max = 3; + int64 Min = 4; + int64 Avg = 5; + int64 Count = 6; +} + message ExecuteQueryResult { Ydb.ResultSet result = 1; - bytes yson = 2; -} - -message TFile { - enum EFileType { - EUDF_FILE = 0; - EUSER_FILE = 1; - EEXE_FILE = 2; - } - - string LocalPath = 1; - string ObjectId = 2; - string Name = 3; - EFileType ObjectType = 4; - int64 Size = 5; -} - -message TWorkerInfo { - // TGuid Guid = 1; // reserved - uint32 NodeId = 2; - string ClusterName = 3; - repeated TAttribute Attribute = 4; - string JobId = 5; - string OperationId = 6; - string Guid = 7; -}; - -message TTaskMeta { - uint64 StageId = 1; - repeated TFile Files = 2; - map<string, string> SecureParams = 3; - map<string, bytes> TaskParams = 4; - repeated TAttr Settings = 5; // pragmas - repeated TWorkerInfo WorkerInfo = 6; //debug info - string ClusterNameHint = 7; -} - -message TWorkerFilter { - repeated TFile File = 1; - string Revision = 2; - string ClusterName = 3; - string ClusterNameHint = 4; - repeated string Address = 5; - repeated uint64 NodeId = 6; - repeated uint64 NodeIdHint = 7; -} - -message ExecuteGraphRequest { - Ydb.Operations.OperationParams Params = 1; - repeated NYql.NDqProto.TDqTask Task = 3; - uint64 SourceId = 4; - bytes ResultType = 5; - repeated string Columns = 6; - string Session = 7; - repeated TAttr Settings = 8; + bytes yson = 2; +} + +message TFile { + enum EFileType { + EUDF_FILE = 0; + EUSER_FILE = 1; + EEXE_FILE = 2; + } + + string LocalPath = 1; + string ObjectId = 2; + string Name = 3; + EFileType ObjectType = 4; + int64 Size = 5; +} + +message TWorkerInfo { + // TGuid Guid = 1; // reserved + uint32 NodeId = 2; + string ClusterName = 3; + repeated TAttribute Attribute = 4; + string JobId = 5; + string OperationId = 6; + string Guid = 7; +}; + +message TTaskMeta { + uint64 StageId = 1; + repeated TFile Files = 2; + map<string, string> SecureParams = 3; + map<string, bytes> TaskParams = 4; + repeated TAttr Settings = 5; // pragmas + repeated TWorkerInfo WorkerInfo = 6; //debug info + string ClusterNameHint = 7; +} + +message TWorkerFilter { + repeated TFile File = 1; + string Revision = 2; + string ClusterName = 3; + string ClusterNameHint = 4; + repeated string Address = 5; + repeated uint64 NodeId = 6; + repeated uint64 NodeIdHint = 7; +} + +message ExecuteGraphRequest { + Ydb.Operations.OperationParams Params = 1; + repeated NYql.NDqProto.TDqTask Task = 3; + uint64 SourceId = 4; + bytes ResultType = 5; + repeated string Columns = 6; + string Session = 7; + repeated TAttr Settings = 8; map<string, string> SecureParams = 9; bool Discard = 12; map<string, string> GraphParams = 13; -} - -message ExecuteGraphResponse { - Ydb.Operations.Operation Operation = 1; - repeated ResponseMetric Metric = 2; - bool Truncated = 3; -} - +} + +message ExecuteGraphResponse { + Ydb.Operations.Operation Operation = 1; + repeated ResponseMetric Metric = 2; + bool Truncated = 3; +} + message SvnRevisionRequest { } message SvnRevisionResponse { string Revision = 1; } - -message OpenSessionRequest { - string Session = 1; + +message OpenSessionRequest { + string Session = 1; string Username = 2; -} - -message OpenSessionResponse { -} - -message CloseSessionRequest { - string Session = 1; -} - -message CloseSessionResponse { -} - -message PingSessionRequest { - string Session = 1; -} - -message PingSessionResponse { -} - +} + +message OpenSessionResponse { +} + +message CloseSessionRequest { + string Session = 1; +} + +message CloseSessionResponse { +} + +message PingSessionRequest { + string Session = 1; +} + +message PingSessionResponse { +} + message QueryStatusRequest { string Session = 1; } @@ -126,179 +126,179 @@ message QueryStatusResponse { string Status = 1; } -message TAttribute { - string Key = 1; - string Value = 2; -} - -message RegisterNodeRequest { - message TGuid { - fixed32 Dw0 = 1; - fixed32 Dw1 = 2; - fixed32 Dw2 = 3; - fixed32 Dw3 = 4; - } - - TGuid Guid = 11; - - uint32 NodeId = 1; - string Address = 2; - uint32 Port = 3; - string Role = 4; - repeated uint32 KnownNodes = 5; - string Revision = 6; - string ClusterName = 7; - uint32 RunningWorkers = 8; - repeated string RunningOperation = 22; - uint32 Epoch = 9; - - message LocalFile { - string ObjectId = 2; - } - repeated LocalFile FilesOnNode = 10; - repeated TFile DownloadList = 14; // filled authomatically - - repeated TAttribute Attribute = 12; - - enum TCapability { - ECAP_NONE = 0; - ECAP_RUNEXE = 1; - ECAP_COMPUTE_ACTOR = 2; - } - - uint32 Capabilities = 15; - int64 FreeDiskSize = 16; - int64 UsedDiskSize = 17; - uint32 Capacity = 18; - string StartTime = 19; - bool Zombie = 20; // set by gwm - - message TRusage { - int64 Stime = 1; - int64 Utime = 2; - int64 MajorPageFaults = 6; - int64 CpuSystem = 7; - int64 CpuUser = 8; - int64 CpuTotal = 9; - } - - TRusage Rusage = 21; - - bool Full = 23; // If Full == false only Guid? present - uint32 Pid = 24; -} - -message RegisterNodeResponse { - message Node { - uint32 NodeId = 1; - string Address = 2; - uint32 Port = 3; - } - - repeated Node Nodes = 1; - uint32 Epoch = 2; - repeated TFile DownloadList = 3; - - string Revision = 4; -} - -message JobStopRequest { - string WorkerId = 1; - string Revision = 2; - string ClusterName = 3; - repeated TAttribute Attribute = 4; - bool Force = 5; - bool NegativeRevision = 6; -} - -message JobStopResponse { -} - -message OperationStopRequest { - string OperationId = 1; -} - -message OperationStopResponse { -} - -message ClusterStatusRequest { -} - -message ClusterStatusResponse { - message File { - string ObjectId = 2; - string Name = 3; - int32 Count = 4; - int32 Left = 5; - int64 Size = 6; - } - - message TWorker { - string Guid = 1; - uint32 NodeId = 2; - string Address = 3; - uint32 Port = 4; - uint32 Epoch = 5; - string LastPingTime = 6; - string Revision = 7; - string ClusterName = 8; - repeated string Resource = 9; - repeated File DownloadList = 10; - repeated TFile ActiveDownload = 22; - uint64 UseCount = 12; - int64 UseTime = 13; - repeated TAttribute Attribute = 14; - bool Stopping = 15; - int64 FreeDiskSize = 17; - int64 UsedDiskSize = 18; - uint32 Capacity = 19; - int32 RunningWorkerActors = 20; - repeated string Operation = 26; - int32 RunningRequests = 24; - string StartTime = 21; - bool Dead = 23; - RegisterNodeRequest.TRusage Rusage = 25; - } - - string Revision = 1; - repeated TWorker Worker = 2; - - message WaitInfo { - uint32 Count = 1; - repeated File Resources = 2; - string OperationId = 3; - string UserName = 4; - } - repeated WaitInfo WaitList = 4; - - message ResourceStat { - string Id = 1; - string Name = 6; - TFile.EFileType ObjectType = 7; - int64 Size = 8; - int64 UseTime = 2; - int64 WaitTime = 3; - int64 UseCount = 4; - int64 TryCount = 5; - } - - repeated ResourceStat Resource = 5; - repeated string Uploading = 6; - int64 FreeListSize = 7; - int64 Capacity = 8; - int64 RunningRequests = 9; - - message TServiceNode { - string Guid = 1; - uint32 NodeId = 2; - string Revision = 3; - repeated string File = 4; - uint32 Pid = 5; - string Address = 6; - }; - - repeated TServiceNode ServiceNode = 10; -} +message TAttribute { + string Key = 1; + string Value = 2; +} + +message RegisterNodeRequest { + message TGuid { + fixed32 Dw0 = 1; + fixed32 Dw1 = 2; + fixed32 Dw2 = 3; + fixed32 Dw3 = 4; + } + + TGuid Guid = 11; + + uint32 NodeId = 1; + string Address = 2; + uint32 Port = 3; + string Role = 4; + repeated uint32 KnownNodes = 5; + string Revision = 6; + string ClusterName = 7; + uint32 RunningWorkers = 8; + repeated string RunningOperation = 22; + uint32 Epoch = 9; + + message LocalFile { + string ObjectId = 2; + } + repeated LocalFile FilesOnNode = 10; + repeated TFile DownloadList = 14; // filled authomatically + + repeated TAttribute Attribute = 12; + + enum TCapability { + ECAP_NONE = 0; + ECAP_RUNEXE = 1; + ECAP_COMPUTE_ACTOR = 2; + } + + uint32 Capabilities = 15; + int64 FreeDiskSize = 16; + int64 UsedDiskSize = 17; + uint32 Capacity = 18; + string StartTime = 19; + bool Zombie = 20; // set by gwm + + message TRusage { + int64 Stime = 1; + int64 Utime = 2; + int64 MajorPageFaults = 6; + int64 CpuSystem = 7; + int64 CpuUser = 8; + int64 CpuTotal = 9; + } + + TRusage Rusage = 21; + + bool Full = 23; // If Full == false only Guid? present + uint32 Pid = 24; +} + +message RegisterNodeResponse { + message Node { + uint32 NodeId = 1; + string Address = 2; + uint32 Port = 3; + } + + repeated Node Nodes = 1; + uint32 Epoch = 2; + repeated TFile DownloadList = 3; + + string Revision = 4; +} + +message JobStopRequest { + string WorkerId = 1; + string Revision = 2; + string ClusterName = 3; + repeated TAttribute Attribute = 4; + bool Force = 5; + bool NegativeRevision = 6; +} + +message JobStopResponse { +} + +message OperationStopRequest { + string OperationId = 1; +} + +message OperationStopResponse { +} + +message ClusterStatusRequest { +} + +message ClusterStatusResponse { + message File { + string ObjectId = 2; + string Name = 3; + int32 Count = 4; + int32 Left = 5; + int64 Size = 6; + } + + message TWorker { + string Guid = 1; + uint32 NodeId = 2; + string Address = 3; + uint32 Port = 4; + uint32 Epoch = 5; + string LastPingTime = 6; + string Revision = 7; + string ClusterName = 8; + repeated string Resource = 9; + repeated File DownloadList = 10; + repeated TFile ActiveDownload = 22; + uint64 UseCount = 12; + int64 UseTime = 13; + repeated TAttribute Attribute = 14; + bool Stopping = 15; + int64 FreeDiskSize = 17; + int64 UsedDiskSize = 18; + uint32 Capacity = 19; + int32 RunningWorkerActors = 20; + repeated string Operation = 26; + int32 RunningRequests = 24; + string StartTime = 21; + bool Dead = 23; + RegisterNodeRequest.TRusage Rusage = 25; + } + + string Revision = 1; + repeated TWorker Worker = 2; + + message WaitInfo { + uint32 Count = 1; + repeated File Resources = 2; + string OperationId = 3; + string UserName = 4; + } + repeated WaitInfo WaitList = 4; + + message ResourceStat { + string Id = 1; + string Name = 6; + TFile.EFileType ObjectType = 7; + int64 Size = 8; + int64 UseTime = 2; + int64 WaitTime = 3; + int64 UseCount = 4; + int64 TryCount = 5; + } + + repeated ResourceStat Resource = 5; + repeated string Uploading = 6; + int64 FreeListSize = 7; + int64 Capacity = 8; + int64 RunningRequests = 9; + + message TServiceNode { + string Guid = 1; + uint32 NodeId = 2; + string Revision = 3; + repeated string File = 4; + uint32 Pid = 5; + string Address = 6; + }; + + repeated TServiceNode ServiceNode = 10; +} message GetMasterRequest { } @@ -326,22 +326,22 @@ message IsReadyRequest { message IsReadyResponse { bool IsReady = 1; } - -message RoutesRequest { - uint32 NodeId = 1; -} - -message RoutesResponse { - repeated RegisterNodeResponse.Node Nodes = 1; -} - -message BenchmarkRequest { - int32 WorkerCount = 1; - int32 Inflight = 2; - int32 TotalRequests = 3; - int64 MaxRunTimeMs = 4; -} - -message BenchmarkResponse { - // to be filled -} + +message RoutesRequest { + uint32 NodeId = 1; +} + +message RoutesResponse { + repeated RegisterNodeResponse.Node Nodes = 1; +} + +message BenchmarkRequest { + int32 WorkerCount = 1; + int32 Inflight = 2; + int32 TotalRequests = 3; + int64 MaxRunTimeMs = 4; +} + +message BenchmarkResponse { + // to be filled +} diff --git a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto index 5a79c483e4..6e3d08e595 100644 --- a/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto +++ b/ydb/library/yql/providers/dq/api/protos/task_command_executor.proto @@ -1,34 +1,34 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - -package NYql.NDqProto; - +syntax = "proto3"; +option cc_enable_arenas = true; + +package NYql.NDqProto; + import "ydb/library/yql/dq/actors/protos/dq_stats.proto"; import "ydb/library/yql/dq/proto/dq_transport.proto"; import "ydb/library/yql/dq/proto/dq_tasks.proto"; import "ydb/library/yql/providers/dq/api/protos/dqs.proto"; - -message TCommandHeader { - enum ECommand { - VERSION = 0; - PUSH = 1; - FINISH = 2; - POP = 3; - IS_FINISHED = 4; - RUN = 5; - PREPARE = 6; - STOP = 7; - GET_INPUT_TYPE = 8; - GET_FREE_SPACE = 9; - - FINISH_OUTPUT = 10; - DROP_OUTPUT = 11; - TERMINATE_OUTPUT = 12; - GET_STORED_BYTES = 13; - - GET_STATS = 14; - GET_STATS_INPUT = 15; - GET_STATS_OUTPUT = 16; + +message TCommandHeader { + enum ECommand { + VERSION = 0; + PUSH = 1; + FINISH = 2; + POP = 3; + IS_FINISHED = 4; + RUN = 5; + PREPARE = 6; + STOP = 7; + GET_INPUT_TYPE = 8; + GET_FREE_SPACE = 9; + + FINISH_OUTPUT = 10; + DROP_OUTPUT = 11; + TERMINATE_OUTPUT = 12; + GET_STORED_BYTES = 13; + + GET_STATS = 14; + GET_STATS_INPUT = 15; + GET_STATS_OUTPUT = 16; GET_STATS_SOURCE = 17; GET_FREE_SPACE_SOURCE = 18; @@ -36,113 +36,113 @@ message TCommandHeader { PUSH_SOURCE = 20; GET_SOURCE_TYPE = 21; FINISH_SOURCE = 22; - - // Sink - SINK_POP = 23; // TSinkPopRequest -> TSinkPopResponse - SINK_IS_FINISHED = 24; // Header -> TIsFinishedResponse - SINK_OUTPUT_TYPE = 25; // Header -> TGetTypeResponse - SINK_STATS = 26; // Header -> TGetSinkStatsResponse - }; - - int32 Version = 1; - ECommand Command = 2; - uint64 TaskId = 3; - uint64 ChannelId = 4; -} - -message TGetVersionResponse { - int32 Version = 1; -} - -message TGetFreeSpaceResponse { - int64 FreeSpace = 1; -} - -message TPopResponse { - bool Result = 1; - TData Data = 2; - repeated TMetric Metric = 3; -} - -message TSinkPopRequest { - uint64 Bytes = 1; - bool Raw = 2; -}; - -message TSinkPopResponse { - uint64 Bytes = 1; - TData Data = 2; - repeated TMetric Metric = 3; - repeated bytes String = 4; -} - -message TIsFinishedResponse { - bool Result = 1; -} - -message TRunResponse { - int32 Result = 1; - repeated TMetric Metric = 2; - TRusage Rusage = 3; -} - -message TPrepareResponse { - bool Result = 1; // unused - repeated TMetric Metric = 2; -} - + + // Sink + SINK_POP = 23; // TSinkPopRequest -> TSinkPopResponse + SINK_IS_FINISHED = 24; // Header -> TIsFinishedResponse + SINK_OUTPUT_TYPE = 25; // Header -> TGetTypeResponse + SINK_STATS = 26; // Header -> TGetSinkStatsResponse + }; + + int32 Version = 1; + ECommand Command = 2; + uint64 TaskId = 3; + uint64 ChannelId = 4; +} + +message TGetVersionResponse { + int32 Version = 1; +} + +message TGetFreeSpaceResponse { + int64 FreeSpace = 1; +} + +message TPopResponse { + bool Result = 1; + TData Data = 2; + repeated TMetric Metric = 3; +} + +message TSinkPopRequest { + uint64 Bytes = 1; + bool Raw = 2; +}; + +message TSinkPopResponse { + uint64 Bytes = 1; + TData Data = 2; + repeated TMetric Metric = 3; + repeated bytes String = 4; +} + +message TIsFinishedResponse { + bool Result = 1; +} + +message TRunResponse { + int32 Result = 1; + repeated TMetric Metric = 2; + TRusage Rusage = 3; +} + +message TPrepareResponse { + bool Result = 1; // unused + repeated TMetric Metric = 2; +} + message TGetTypeResponse { - bytes Result = 1; -} - -message TDropOutputResponse { - uint64 Result = 1; -} - -message TGetStoredBytesResponse { - uint64 Result = 1; -} - -message TGetStatsResponse { + bytes Result = 1; +} + +message TDropOutputResponse { + uint64 Result = 1; +} + +message TGetStoredBytesResponse { + uint64 Result = 1; +} + +message TGetStatsResponse { reserved 1; // NKqpProto.TKqpStatsTask Stats = 1; NDqProto.TDqTaskStats Stats = 2; -} - -message TGetStatsInputResponse { +} + +message TGetStatsInputResponse { reserved 1; // NKqpProto.TKqpStatsTask.TInputChannelStats Stats = 1; NDqProto.TDqInputChannelStats Stats = 2; -} - -message TGetStatsOutputResponse { +} + +message TGetStatsOutputResponse { reserved 1; // NKqpProto.TKqpStatsTask.TOutputChannelStats Stats = 1; NDqProto.TDqOutputChannelStats Stats = 2; -} - +} + message TGetStatsSourceResponse { reserved 1; // NKqpProto.TKqpStatsTask.TSourceStats Stats = 1; NDqProto.TDqSourceStats Stats = 2; } -message TSinkStatsResponse { - NDqProto.TDqSinkStats Stats = 1; -} - -message TPrepareRequest { - NYql.NDqProto.TDqTask Task = 1; -} +message TSinkStatsResponse { + NDqProto.TDqSinkStats Stats = 1; +} + +message TPrepareRequest { + NYql.NDqProto.TDqTask Task = 1; +} message TSourcePushRequest { TData Data = 1; - int64 Space = 2; - repeated bytes String = 3; - int32 Chunks = 4; -} - -message TSourcePushChunk { - int64 Parts = 1; - bytes String = 2; -} - -message TSourcePushPart { - bytes String = 1; -} + int64 Space = 2; + repeated bytes String = 3; + int32 Chunks = 4; +} + +message TSourcePushChunk { + int64 Parts = 1; + bytes String = 2; +} + +message TSourcePushPart { + bytes String = 1; +} diff --git a/ydb/library/yql/providers/dq/api/protos/ya.make b/ydb/library/yql/providers/dq/api/protos/ya.make index 7d64cb2ff1..6f48289373 100644 --- a/ydb/library/yql/providers/dq/api/protos/ya.make +++ b/ydb/library/yql/providers/dq/api/protos/ya.make @@ -4,12 +4,12 @@ OWNER(g:yql) SRCS( service.proto - dqs.proto - task_command_executor.proto + dqs.proto + task_command_executor.proto ) PEERDIR( - library/cpp/actors/protos + library/cpp/actors/protos ydb/public/api/protos ydb/library/yql/dq/actors/protos ydb/library/yql/dq/proto diff --git a/ydb/library/yql/providers/dq/backtrace/backtrace.cpp b/ydb/library/yql/providers/dq/backtrace/backtrace.cpp index ebeb62468a..12dc7d2ab9 100644 --- a/ydb/library/yql/providers/dq/backtrace/backtrace.cpp +++ b/ydb/library/yql/providers/dq/backtrace/backtrace.cpp @@ -1,110 +1,110 @@ -#include "backtrace.h" - -#include <library/cpp/malloc/api/malloc.h> - -#include <util/generic/hash.h> -#include <util/generic/string.h> - -#include <util/system/mlock.h> -#include <util/system/backtrace.h> -#include <util/stream/output.h> -#include <util/string/printf.h> - -#ifdef _linux_ -#include <dlfcn.h> -#include <link.h> -#endif - -namespace NYql { - -namespace NBacktrace { - -namespace { - -TAtomic BacktraceStarted = 0; -void* Stack[300]; -THashMap<TString, TString> Mapping; - -struct TDllInfo { - TString Path; - ui64 BaseAddress; -}; - -#ifdef _linux_ -int DlIterCallback(struct dl_phdr_info *info, size_t size, void *data) -{ - Y_UNUSED(size); - Y_UNUSED(data); - if (*info->dlpi_name) { - TDllInfo dllInfo{ info->dlpi_name, (ui64)info->dlpi_addr }; - ((THashMap<TString, TDllInfo>*)data)->emplace(dllInfo.Path, dllInfo); - } - - return 0; -} -#endif - -} /* namespace */ - -void PrintBacktraceToStderr(int signum) -{ - if (!NMalloc::IsAllocatorCorrupted) { - /* we want memory allocation for backtrace printing */ - - if (!AtomicTryLock(&BacktraceStarted)) { - return; - } - - // Unlock memory to avoid extra RSS consumption while touching debug info - UnlockAllMemory(); - - const size_t s = BackTrace(Stack, Y_ARRAY_SIZE(Stack)); - - TString binaryPath = "EXE"; - THashMap<TString, TDllInfo> dlls; -#ifdef _linux_ - dl_iterate_phdr(DlIterCallback, &dlls); -#endif - - Cerr << "StackFrames: " << s << Endl; - - for (size_t i = 0; i < s; ++i) { - ui64 address = (ui64)Stack[i] - 1; // last byte of the call instruction - ui64 offset = 0; - TString modulePath = binaryPath; -#ifdef _linux_ - Dl_info dlInfo; - memset(&dlInfo, 0, sizeof(dlInfo)); - auto ret = dladdr((void*)address, &dlInfo); - if (ret) { - auto path = dlInfo.dli_fname; - auto it = dlls.find(path); - if (it != dlls.end()) { - modulePath = path; - offset = it->second.BaseAddress; - } - } -#endif - - auto it = Mapping.find(modulePath); - if (it != Mapping.end()) { - modulePath = it->second; - } - - Cerr << "StackFrame: " << modulePath << " " << address << " " << offset << Endl; - } - } - /* Now reraise the signal. We reactivate the signal’s default handling, - which is to terminate the process. We could just call exit or abort, - but reraising the signal sets the return status from the process - correctly. */ - raise(signum); -} - -void SetModulesMapping(const THashMap<TString, TString>& mapping) { - Mapping = mapping; -} - -} /* namespace NBacktrace */ - -} /* namespace NYql */ +#include "backtrace.h" + +#include <library/cpp/malloc/api/malloc.h> + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +#include <util/system/mlock.h> +#include <util/system/backtrace.h> +#include <util/stream/output.h> +#include <util/string/printf.h> + +#ifdef _linux_ +#include <dlfcn.h> +#include <link.h> +#endif + +namespace NYql { + +namespace NBacktrace { + +namespace { + +TAtomic BacktraceStarted = 0; +void* Stack[300]; +THashMap<TString, TString> Mapping; + +struct TDllInfo { + TString Path; + ui64 BaseAddress; +}; + +#ifdef _linux_ +int DlIterCallback(struct dl_phdr_info *info, size_t size, void *data) +{ + Y_UNUSED(size); + Y_UNUSED(data); + if (*info->dlpi_name) { + TDllInfo dllInfo{ info->dlpi_name, (ui64)info->dlpi_addr }; + ((THashMap<TString, TDllInfo>*)data)->emplace(dllInfo.Path, dllInfo); + } + + return 0; +} +#endif + +} /* namespace */ + +void PrintBacktraceToStderr(int signum) +{ + if (!NMalloc::IsAllocatorCorrupted) { + /* we want memory allocation for backtrace printing */ + + if (!AtomicTryLock(&BacktraceStarted)) { + return; + } + + // Unlock memory to avoid extra RSS consumption while touching debug info + UnlockAllMemory(); + + const size_t s = BackTrace(Stack, Y_ARRAY_SIZE(Stack)); + + TString binaryPath = "EXE"; + THashMap<TString, TDllInfo> dlls; +#ifdef _linux_ + dl_iterate_phdr(DlIterCallback, &dlls); +#endif + + Cerr << "StackFrames: " << s << Endl; + + for (size_t i = 0; i < s; ++i) { + ui64 address = (ui64)Stack[i] - 1; // last byte of the call instruction + ui64 offset = 0; + TString modulePath = binaryPath; +#ifdef _linux_ + Dl_info dlInfo; + memset(&dlInfo, 0, sizeof(dlInfo)); + auto ret = dladdr((void*)address, &dlInfo); + if (ret) { + auto path = dlInfo.dli_fname; + auto it = dlls.find(path); + if (it != dlls.end()) { + modulePath = path; + offset = it->second.BaseAddress; + } + } +#endif + + auto it = Mapping.find(modulePath); + if (it != Mapping.end()) { + modulePath = it->second; + } + + Cerr << "StackFrame: " << modulePath << " " << address << " " << offset << Endl; + } + } + /* Now reraise the signal. We reactivate the signal’s default handling, + which is to terminate the process. We could just call exit or abort, + but reraising the signal sets the return status from the process + correctly. */ + raise(signum); +} + +void SetModulesMapping(const THashMap<TString, TString>& mapping) { + Mapping = mapping; +} + +} /* namespace NBacktrace */ + +} /* namespace NYql */ diff --git a/ydb/library/yql/providers/dq/backtrace/backtrace.h b/ydb/library/yql/providers/dq/backtrace/backtrace.h index ab53c0e9f8..7e3e33b7e0 100644 --- a/ydb/library/yql/providers/dq/backtrace/backtrace.h +++ b/ydb/library/yql/providers/dq/backtrace/backtrace.h @@ -1,16 +1,16 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/generic/string.h> - -namespace NYql { - -namespace NBacktrace { - -void SetModulesMapping(const THashMap<TString, TString>& mapping); -void PrintBacktraceToStderr(int signum); -TString Symbolize(const TString& input, const THashMap<TString, TString>& mapping); - -} /* namespace Backtrace */ - -} /* namespace NYql */ +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/string.h> + +namespace NYql { + +namespace NBacktrace { + +void SetModulesMapping(const THashMap<TString, TString>& mapping); +void PrintBacktraceToStderr(int signum); +TString Symbolize(const TString& input, const THashMap<TString, TString>& mapping); + +} /* namespace Backtrace */ + +} /* namespace NYql */ diff --git a/ydb/library/yql/providers/dq/backtrace/symbolizer.cpp b/ydb/library/yql/providers/dq/backtrace/symbolizer.cpp index b037d32f32..df97a6aa8a 100644 --- a/ydb/library/yql/providers/dq/backtrace/symbolizer.cpp +++ b/ydb/library/yql/providers/dq/backtrace/symbolizer.cpp @@ -1,117 +1,117 @@ -#include "backtrace.h" - -#ifdef _linux_ -#include <llvm/DebugInfo/Symbolize/Symbolize.h> -#include <llvm/DebugInfo/Symbolize/DIPrinter.h> -#include <llvm/Support/raw_ostream.h> -#endif - -#include <util/string/split.h> -#include <util/stream/str.h> - -namespace NYql { - -namespace NBacktrace { - -struct TStackFrame { - TString ModulePath; - ui64 Address; - ui64 Offset; -}; - -#ifdef _linux_ -class TRawOStreamProxy: public llvm::raw_ostream { -public: - TRawOStreamProxy(IOutputStream& out) - : llvm::raw_ostream(true) // unbuffered - , Slave_(out) - { - } - void write_impl(const char* ptr, size_t size) override { - Slave_.Write(ptr, size); - } - uint64_t current_pos() const override { - return 0; - } - size_t preferred_buffer_size() const override { - return 0; - } -private: - IOutputStream& Slave_; -}; -#endif - -TString Symbolize(const TString& input, const THashMap<TString, TString>& mapping) { - TString output; - TStringOutput out(output); - -#ifdef _linux_ - TRawOStreamProxy outStream(out); - llvm::symbolize::LLVMSymbolizer::Options opts; - llvm::symbolize::LLVMSymbolizer symbolyzer(opts); - llvm::symbolize::DIPrinter printer(outStream, true, true, false); -#else - auto& outStream = out; -#endif - - i64 stackSize = -1; - TVector<TStackFrame> frames; - for (TStringBuf line: StringSplitter(input).SplitByString("\n").SkipEmpty()) { - if (line.StartsWith("StackFrames:")) { - TVector<TString> parts; - Split(TString(line), " ", parts); - if (parts.size() > 1) { - TryFromString<i64>(parts[1], stackSize); - frames.reserve(stackSize); - } - } else if (line.StartsWith("StackFrame:")) { - TVector<TString> parts; - Split(TString(line), " ", parts); - TString modulePath; - ui64 address; - ui64 offset; - if (parts.size() > 3) { - modulePath = parts[1]; - TryFromString<ui64>(parts[2], address); - TryFromString<ui64>(parts[3], offset); - auto it = mapping.find(modulePath); - if (it != mapping.end()) { - modulePath = it->second; - } - frames.emplace_back(TStackFrame{modulePath, address, offset}); - } - } else { - outStream << line << "\n"; - } - } - - if (stackSize == 0) { - outStream << "Empty stack trace\n"; - } - - for (const auto& frame : frames) { -#ifdef _linux_ - llvm::object::SectionedAddress secAddr; - secAddr.Address = frame.Address - frame.Offset; - auto resOrErr = symbolyzer.symbolizeCode(frame.ModulePath, secAddr); - if (resOrErr) { - auto value = resOrErr.get(); - if (value.FileName == "<invalid>" && frame.Offset > 0) { - value.FileName = frame.ModulePath; - } - - printer << value; - } else { - logAllUnhandledErrors(resOrErr.takeError(), outStream, - "LLVMSymbolizer: error reading file: "); - } -#else - outStream << "StackFrame: " << frame.ModulePath << " " << frame.Address << " " << frame.Offset << Endl; -#endif - } - return output; -} - -} /* namespace NBacktrace */ - -} /* namespace NYql */ +#include "backtrace.h" + +#ifdef _linux_ +#include <llvm/DebugInfo/Symbolize/Symbolize.h> +#include <llvm/DebugInfo/Symbolize/DIPrinter.h> +#include <llvm/Support/raw_ostream.h> +#endif + +#include <util/string/split.h> +#include <util/stream/str.h> + +namespace NYql { + +namespace NBacktrace { + +struct TStackFrame { + TString ModulePath; + ui64 Address; + ui64 Offset; +}; + +#ifdef _linux_ +class TRawOStreamProxy: public llvm::raw_ostream { +public: + TRawOStreamProxy(IOutputStream& out) + : llvm::raw_ostream(true) // unbuffered + , Slave_(out) + { + } + void write_impl(const char* ptr, size_t size) override { + Slave_.Write(ptr, size); + } + uint64_t current_pos() const override { + return 0; + } + size_t preferred_buffer_size() const override { + return 0; + } +private: + IOutputStream& Slave_; +}; +#endif + +TString Symbolize(const TString& input, const THashMap<TString, TString>& mapping) { + TString output; + TStringOutput out(output); + +#ifdef _linux_ + TRawOStreamProxy outStream(out); + llvm::symbolize::LLVMSymbolizer::Options opts; + llvm::symbolize::LLVMSymbolizer symbolyzer(opts); + llvm::symbolize::DIPrinter printer(outStream, true, true, false); +#else + auto& outStream = out; +#endif + + i64 stackSize = -1; + TVector<TStackFrame> frames; + for (TStringBuf line: StringSplitter(input).SplitByString("\n").SkipEmpty()) { + if (line.StartsWith("StackFrames:")) { + TVector<TString> parts; + Split(TString(line), " ", parts); + if (parts.size() > 1) { + TryFromString<i64>(parts[1], stackSize); + frames.reserve(stackSize); + } + } else if (line.StartsWith("StackFrame:")) { + TVector<TString> parts; + Split(TString(line), " ", parts); + TString modulePath; + ui64 address; + ui64 offset; + if (parts.size() > 3) { + modulePath = parts[1]; + TryFromString<ui64>(parts[2], address); + TryFromString<ui64>(parts[3], offset); + auto it = mapping.find(modulePath); + if (it != mapping.end()) { + modulePath = it->second; + } + frames.emplace_back(TStackFrame{modulePath, address, offset}); + } + } else { + outStream << line << "\n"; + } + } + + if (stackSize == 0) { + outStream << "Empty stack trace\n"; + } + + for (const auto& frame : frames) { +#ifdef _linux_ + llvm::object::SectionedAddress secAddr; + secAddr.Address = frame.Address - frame.Offset; + auto resOrErr = symbolyzer.symbolizeCode(frame.ModulePath, secAddr); + if (resOrErr) { + auto value = resOrErr.get(); + if (value.FileName == "<invalid>" && frame.Offset > 0) { + value.FileName = frame.ModulePath; + } + + printer << value; + } else { + logAllUnhandledErrors(resOrErr.takeError(), outStream, + "LLVMSymbolizer: error reading file: "); + } +#else + outStream << "StackFrame: " << frame.ModulePath << " " << frame.Address << " " << frame.Offset << Endl; +#endif + } + return output; +} + +} /* namespace NBacktrace */ + +} /* namespace NYql */ diff --git a/ydb/library/yql/providers/dq/common/attrs.cpp b/ydb/library/yql/providers/dq/common/attrs.cpp index 378e5410d5..a99993f28f 100644 --- a/ydb/library/yql/providers/dq/common/attrs.cpp +++ b/ydb/library/yql/providers/dq/common/attrs.cpp @@ -1,25 +1,25 @@ -#include "attrs.h" - - -namespace NYql { - -namespace NCommonAttrs { - TString ACTOR_NODEID_ATTR = "yql_actor_node_id"; - TString ROLE_ATTR = "yql_role"; - TString HOSTNAME_ATTR = "yql_hostname"; - TString GRPCPORT_ATTR = "yql_grpc_port"; - TString REVISION_ATTR = "yql_revision"; - TString INTERCONNECTPORT_ATTR = "yql_interconnect_port"; - TString EPOCH_ATTR = "yql_epoch"; - TString PINGTIME_ATTR = "yql_ping_time"; - - TString OPERATIONID_ATTR = "yql_operation_id"; - TString OPERATIONSIZE_ATTR = "yql_operation_size"; - TString JOBID_ATTR = "yql_job_id"; - - TString CLUSTERNAME_ATTR = "yql_cluster_name"; - - TString UPLOAD_EXECUTABLE_ATTR = "yql_upload_executable"; -} - -} +#include "attrs.h" + + +namespace NYql { + +namespace NCommonAttrs { + TString ACTOR_NODEID_ATTR = "yql_actor_node_id"; + TString ROLE_ATTR = "yql_role"; + TString HOSTNAME_ATTR = "yql_hostname"; + TString GRPCPORT_ATTR = "yql_grpc_port"; + TString REVISION_ATTR = "yql_revision"; + TString INTERCONNECTPORT_ATTR = "yql_interconnect_port"; + TString EPOCH_ATTR = "yql_epoch"; + TString PINGTIME_ATTR = "yql_ping_time"; + + TString OPERATIONID_ATTR = "yql_operation_id"; + TString OPERATIONSIZE_ATTR = "yql_operation_size"; + TString JOBID_ATTR = "yql_job_id"; + + TString CLUSTERNAME_ATTR = "yql_cluster_name"; + + TString UPLOAD_EXECUTABLE_ATTR = "yql_upload_executable"; +} + +} diff --git a/ydb/library/yql/providers/dq/common/attrs.h b/ydb/library/yql/providers/dq/common/attrs.h index 166100b1eb..123fd104e4 100644 --- a/ydb/library/yql/providers/dq/common/attrs.h +++ b/ydb/library/yql/providers/dq/common/attrs.h @@ -1,24 +1,24 @@ -#pragma once - -#include <util/generic/string.h> - -namespace NYql { -namespace NCommonAttrs { - extern TString ACTOR_NODEID_ATTR; - extern TString ROLE_ATTR; - extern TString HOSTNAME_ATTR; - extern TString GRPCPORT_ATTR; - extern TString REVISION_ATTR; - extern TString INTERCONNECTPORT_ATTR; - extern TString EPOCH_ATTR; - extern TString PINGTIME_ATTR; - - extern TString OPERATIONID_ATTR; - extern TString OPERATIONSIZE_ATTR; - extern TString JOBID_ATTR; - - extern TString CLUSTERNAME_ATTR; - - extern TString UPLOAD_EXECUTABLE_ATTR; -} -} +#pragma once + +#include <util/generic/string.h> + +namespace NYql { +namespace NCommonAttrs { + extern TString ACTOR_NODEID_ATTR; + extern TString ROLE_ATTR; + extern TString HOSTNAME_ATTR; + extern TString GRPCPORT_ATTR; + extern TString REVISION_ATTR; + extern TString INTERCONNECTPORT_ATTR; + extern TString EPOCH_ATTR; + extern TString PINGTIME_ATTR; + + extern TString OPERATIONID_ATTR; + extern TString OPERATIONSIZE_ATTR; + extern TString JOBID_ATTR; + + extern TString CLUSTERNAME_ATTR; + + extern TString UPLOAD_EXECUTABLE_ATTR; +} +} diff --git a/ydb/library/yql/providers/dq/common/ya.make b/ydb/library/yql/providers/dq/common/ya.make index 3bce6ba87d..68d9080dee 100644 --- a/ydb/library/yql/providers/dq/common/ya.make +++ b/ydb/library/yql/providers/dq/common/ya.make @@ -11,9 +11,9 @@ PEERDIR( ) SRCS( - attrs.cpp - yql_dq_common.cpp - yql_dq_settings.cpp + attrs.cpp + yql_dq_common.cpp + yql_dq_settings.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/dq/common/yql_dq_common.cpp b/ydb/library/yql/providers/dq/common/yql_dq_common.cpp index 7559361e33..bd33963442 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_common.cpp +++ b/ydb/library/yql/providers/dq/common/yql_dq_common.cpp @@ -1,71 +1,71 @@ -#include "yql_dq_common.h" - +#include "yql_dq_common.h" + #include <ydb/library/yql/utils/yql_panic.h> - + #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> - + #include <ydb/library/yql/sql/sql.h> #include <ydb/library/yql/sql/settings/translation_settings.h> - + #include <util/string/split.h> -namespace NYql { -namespace NCommon { - -using namespace NKikimr::NMiniKQL; - -TString GetSerializedResultType(const TString& program) { - TScopedAlloc alloc; - TTypeEnvironment typeEnv(alloc); - - TRuntimeNode programNode = DeserializeRuntimeNode(program, typeEnv); - - YQL_ENSURE(programNode.IsImmediate() && programNode.GetNode()->GetType()->IsStruct()); - - // copy-paste from dq_task_runner.cpp - auto& programStruct = static_cast<TStructLiteral&>(*programNode.GetNode()); - auto programType = programStruct.GetType(); - YQL_ENSURE(programType); - - auto programRootIdx = programType->FindMemberIndex("Program"); - YQL_ENSURE(programRootIdx); - TRuntimeNode programRoot = programStruct.GetValue(*programRootIdx); - YQL_ENSURE(programRoot.GetNode()->GetType()->IsCallable()); - auto programResultType = static_cast<const TCallableType*>(programRoot.GetNode()->GetType()); - YQL_ENSURE(programResultType->GetReturnType()->IsStream()); - auto programResultItemType = static_cast<const TStreamType*>(programResultType->GetReturnType())->GetItemType(); - - return SerializeNode(programResultItemType, typeEnv); -} - -TMaybe<TString> SqlToSExpr(const TString& query) { - NSQLTranslation::TTranslationSettings settings; - settings.SyntaxVersion = 1; - settings.Mode = NSQLTranslation::ESqlMode::QUERY; - settings.DefaultCluster = "undefined"; - settings.ClusterMapping[settings.DefaultCluster] = "undefined"; - settings.ClusterMapping["csv"] = "csv"; - settings.ClusterMapping["memory"] = "memory"; - settings.ClusterMapping["ydb"] = "ydb"; - settings.EnableGenericUdfs = true; - settings.File = "generated.sql"; - - auto astRes = NSQLTranslation::SqlToYql(query, settings); - if (!astRes.Issues.Empty()) { - Cerr << astRes.Issues.ToString() << Endl; - } - - if (!astRes.Root) { - return {}; - } - - TStringStream sexpr; - astRes.Root->PrintTo(sexpr); - return sexpr.Str(); -} - +namespace NYql { +namespace NCommon { + +using namespace NKikimr::NMiniKQL; + +TString GetSerializedResultType(const TString& program) { + TScopedAlloc alloc; + TTypeEnvironment typeEnv(alloc); + + TRuntimeNode programNode = DeserializeRuntimeNode(program, typeEnv); + + YQL_ENSURE(programNode.IsImmediate() && programNode.GetNode()->GetType()->IsStruct()); + + // copy-paste from dq_task_runner.cpp + auto& programStruct = static_cast<TStructLiteral&>(*programNode.GetNode()); + auto programType = programStruct.GetType(); + YQL_ENSURE(programType); + + auto programRootIdx = programType->FindMemberIndex("Program"); + YQL_ENSURE(programRootIdx); + TRuntimeNode programRoot = programStruct.GetValue(*programRootIdx); + YQL_ENSURE(programRoot.GetNode()->GetType()->IsCallable()); + auto programResultType = static_cast<const TCallableType*>(programRoot.GetNode()->GetType()); + YQL_ENSURE(programResultType->GetReturnType()->IsStream()); + auto programResultItemType = static_cast<const TStreamType*>(programResultType->GetReturnType())->GetItemType(); + + return SerializeNode(programResultItemType, typeEnv); +} + +TMaybe<TString> SqlToSExpr(const TString& query) { + NSQLTranslation::TTranslationSettings settings; + settings.SyntaxVersion = 1; + settings.Mode = NSQLTranslation::ESqlMode::QUERY; + settings.DefaultCluster = "undefined"; + settings.ClusterMapping[settings.DefaultCluster] = "undefined"; + settings.ClusterMapping["csv"] = "csv"; + settings.ClusterMapping["memory"] = "memory"; + settings.ClusterMapping["ydb"] = "ydb"; + settings.EnableGenericUdfs = true; + settings.File = "generated.sql"; + + auto astRes = NSQLTranslation::SqlToYql(query, settings); + if (!astRes.Issues.Empty()) { + Cerr << astRes.Issues.ToString() << Endl; + } + + if (!astRes.Root) { + return {}; + } + + TStringStream sexpr; + astRes.Root->PrintTo(sexpr); + return sexpr.Str(); +} + bool ParseCounterName(TString* prefix, std::map<TString, TString>* labels, TString* name, const TString& counterName) { auto pos = counterName.find(":"); if (pos == TString::npos) { @@ -74,7 +74,7 @@ bool ParseCounterName(TString* prefix, std::map<TString, TString>* labels, TStri *prefix = counterName.substr(0, pos); auto labelsString = counterName.substr(pos+1); - + *name = ""; for (const auto& kv : StringSplitter(labelsString).Split(',')) { TStringBuf key, value; @@ -85,13 +85,13 @@ bool ParseCounterName(TString* prefix, std::map<TString, TString>* labels, TStri *name = value; } else { (*labels)[TString(key)] = TString(value); - } - } - } - + } + } + } + return !name->empty(); -} - +} + -} // namespace NCommon -} // namespace NYql +} // namespace NCommon +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/common/yql_dq_common.h b/ydb/library/yql/providers/dq/common/yql_dq_common.h index 5646f4b32f..c1c865a7e5 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_common.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_common.h @@ -1,16 +1,16 @@ -#pragma once - -#include <util/generic/string.h> +#pragma once + +#include <util/generic/string.h> #include <map> - -namespace NYql { -namespace NCommon { - -TMaybe<TString> SqlToSExpr(const TString& query); - -TString GetSerializedResultType(const TString& program); - + +namespace NYql { +namespace NCommon { + +TMaybe<TString> SqlToSExpr(const TString& query); + +TString GetSerializedResultType(const TString& program); + bool ParseCounterName(TString* prefix, std::map<TString, TString>* labels, TString* name, const TString& counterName); - -} // namespace NCommon -} // namespace NYql + +} // namespace NCommon +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp index d06f6b4cdd..f9ba89a33d 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp @@ -1,50 +1,50 @@ -#include "yql_dq_settings.h" - -namespace NYql { - -TDqConfiguration::TDqConfiguration() { - REGISTER_SETTING(*this, DataSizePerJob); - REGISTER_SETTING(*this, MaxDataSizePerJob); - REGISTER_SETTING(*this, MaxTasksPerStage); - REGISTER_SETTING(*this, MaxTasksPerOperation); - REGISTER_SETTING(*this, WorkersPerOperation); - REGISTER_SETTING(*this, MaxDataSizePerQuery); +#include "yql_dq_settings.h" + +namespace NYql { + +TDqConfiguration::TDqConfiguration() { + REGISTER_SETTING(*this, DataSizePerJob); + REGISTER_SETTING(*this, MaxDataSizePerJob); + REGISTER_SETTING(*this, MaxTasksPerStage); + REGISTER_SETTING(*this, MaxTasksPerOperation); + REGISTER_SETTING(*this, WorkersPerOperation); + REGISTER_SETTING(*this, MaxDataSizePerQuery); REGISTER_SETTING(*this, AnalyticsHopping); - REGISTER_SETTING(*this, AnalyzeQuery); - REGISTER_SETTING(*this, _AnalyzeQueryPercentage); - REGISTER_SETTING(*this, MaxRetries); - REGISTER_SETTING(*this, MaxNetworkRetries); - REGISTER_SETTING(*this, RetryBackoffMs); + REGISTER_SETTING(*this, AnalyzeQuery); + REGISTER_SETTING(*this, _AnalyzeQueryPercentage); + REGISTER_SETTING(*this, MaxRetries); + REGISTER_SETTING(*this, MaxNetworkRetries); + REGISTER_SETTING(*this, RetryBackoffMs); REGISTER_SETTING(*this, CollectCoreDumps); - REGISTER_SETTING(*this, FallbackPolicy); - REGISTER_SETTING(*this, PullRequestTimeoutMs); - REGISTER_SETTING(*this, PingTimeoutMs); + REGISTER_SETTING(*this, FallbackPolicy); + REGISTER_SETTING(*this, PullRequestTimeoutMs); + REGISTER_SETTING(*this, PingTimeoutMs); REGISTER_SETTING(*this, UseSimpleYtReader); REGISTER_SETTING(*this, OptLLVM); - REGISTER_SETTING(*this, ChannelBufferSize); - REGISTER_SETTING(*this, OutputChunkMaxSize); + REGISTER_SETTING(*this, ChannelBufferSize); + REGISTER_SETTING(*this, OutputChunkMaxSize); REGISTER_SETTING(*this, MemoryLimit); REGISTER_SETTING(*this, EnableInsert); - - REGISTER_SETTING(*this, _LiteralTimeout); - REGISTER_SETTING(*this, _TableTimeout); - REGISTER_SETTING(*this, _LongWorkersAllocationWarnTimeout); - REGISTER_SETTING(*this, _LongWorkersAllocationFailTimeout); - - REGISTER_SETTING(*this, _AllResultsBytesLimit); - REGISTER_SETTING(*this, _RowsLimitPerWrite); - - REGISTER_SETTING(*this, EnableStrip); - - REGISTER_SETTING(*this, EnableComputeActor); - REGISTER_SETTING(*this, ComputeActorType); - REGISTER_SETTING(*this, EnablePorto); - REGISTER_SETTING(*this, _PortoMemoryLimit); - - REGISTER_SETTING(*this, EnableFullResultWrite); - - REGISTER_SETTING(*this, _FallbackOnRuntimeErrors); - REGISTER_SETTING(*this, WorkerFilter); -} - -} // namespace NYql + + REGISTER_SETTING(*this, _LiteralTimeout); + REGISTER_SETTING(*this, _TableTimeout); + REGISTER_SETTING(*this, _LongWorkersAllocationWarnTimeout); + REGISTER_SETTING(*this, _LongWorkersAllocationFailTimeout); + + REGISTER_SETTING(*this, _AllResultsBytesLimit); + REGISTER_SETTING(*this, _RowsLimitPerWrite); + + REGISTER_SETTING(*this, EnableStrip); + + REGISTER_SETTING(*this, EnableComputeActor); + REGISTER_SETTING(*this, ComputeActorType); + REGISTER_SETTING(*this, EnablePorto); + REGISTER_SETTING(*this, _PortoMemoryLimit); + + REGISTER_SETTING(*this, EnableFullResultWrite); + + REGISTER_SETTING(*this, _FallbackOnRuntimeErrors); + REGISTER_SETTING(*this, WorkerFilter); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h index 0d8b554850..3c37e4d48f 100644 --- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h +++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h @@ -1,158 +1,158 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/common/config/yql_dispatch.h> #include <ydb/library/yql/providers/common/config/yql_setting.h> - + #include <ydb/library/yql/core/yql_data_provider.h> - + #include <library/cpp/string_utils/parse_size/parse_size.h> -#include <util/generic/size_literals.h> -#include <util/random/random.h> - -namespace NYql { - -struct TDqSettings { +#include <util/generic/size_literals.h> +#include <util/random/random.h> + +namespace NYql { + +struct TDqSettings { struct TDefault { static constexpr ui32 MaxTasksPerStage = 20U; static constexpr ui32 MaxTasksPerOperation = 70U; - static constexpr ui64 PortoMemoryLimit = 3_GB; - static constexpr bool EnablePorto = false; + static constexpr ui64 PortoMemoryLimit = 3_GB; + static constexpr bool EnablePorto = false; static constexpr ui64 DataSizePerJob = 128_MB; - static constexpr ui64 MaxDataSizePerJob = 600_MB; - static constexpr int MaxNetworkRetries = 5; - static constexpr ui64 LiteralTimeout = 60000; // 1 minutes - static constexpr ui64 TableTimeout = 600000; // 10 minutes + static constexpr ui64 MaxDataSizePerJob = 600_MB; + static constexpr int MaxNetworkRetries = 5; + static constexpr ui64 LiteralTimeout = 60000; // 1 minutes + static constexpr ui64 TableTimeout = 600000; // 10 minutes static constexpr ui32 CloudFunctionConcurrency = 10; }; using TPtr = std::shared_ptr<TDqSettings>; - - NCommon::TConfSetting<ui64, false> DataSizePerJob; - NCommon::TConfSetting<ui64, false> MaxDataSizePerJob; - NCommon::TConfSetting<ui32, false> MaxTasksPerStage; - NCommon::TConfSetting<ui32, false> MaxTasksPerOperation; - NCommon::TConfSetting<ui32, false> WorkersPerOperation; - NCommon::TConfSetting<ui64, false> MaxDataSizePerQuery; + + NCommon::TConfSetting<ui64, false> DataSizePerJob; + NCommon::TConfSetting<ui64, false> MaxDataSizePerJob; + NCommon::TConfSetting<ui32, false> MaxTasksPerStage; + NCommon::TConfSetting<ui32, false> MaxTasksPerOperation; + NCommon::TConfSetting<ui32, false> WorkersPerOperation; + NCommon::TConfSetting<ui64, false> MaxDataSizePerQuery; NCommon::TConfSetting<bool, false> AnalyticsHopping; - NCommon::TConfSetting<bool, false> AnalyzeQuery; - NCommon::TConfSetting<int, false> _AnalyzeQueryPercentage; - NCommon::TConfSetting<int, false> MaxRetries; - NCommon::TConfSetting<int, false> MaxNetworkRetries; - NCommon::TConfSetting<ui64, false> RetryBackoffMs; + NCommon::TConfSetting<bool, false> AnalyzeQuery; + NCommon::TConfSetting<int, false> _AnalyzeQueryPercentage; + NCommon::TConfSetting<int, false> MaxRetries; + NCommon::TConfSetting<int, false> MaxNetworkRetries; + NCommon::TConfSetting<ui64, false> RetryBackoffMs; NCommon::TConfSetting<bool, false> CollectCoreDumps; - NCommon::TConfSetting<TString, false> FallbackPolicy; - NCommon::TConfSetting<ui64, false> PullRequestTimeoutMs; - NCommon::TConfSetting<ui64, false> PingTimeoutMs; + NCommon::TConfSetting<TString, false> FallbackPolicy; + NCommon::TConfSetting<ui64, false> PullRequestTimeoutMs; + NCommon::TConfSetting<ui64, false> PingTimeoutMs; NCommon::TConfSetting<bool, false> UseSimpleYtReader; NCommon::TConfSetting<TString, false> OptLLVM; - NCommon::TConfSetting<ui64, false> ChannelBufferSize; - NCommon::TConfSetting<ui64, false> OutputChunkMaxSize; + NCommon::TConfSetting<ui64, false> ChannelBufferSize; + NCommon::TConfSetting<ui64, false> OutputChunkMaxSize; NCommon::TConfSetting<NSize::TSize, false> MemoryLimit; - NCommon::TConfSetting<ui64, false> _LiteralTimeout; - NCommon::TConfSetting<ui64, false> _TableTimeout; - NCommon::TConfSetting<ui64, false> _LongWorkersAllocationWarnTimeout; - NCommon::TConfSetting<ui64, false> _LongWorkersAllocationFailTimeout; + NCommon::TConfSetting<ui64, false> _LiteralTimeout; + NCommon::TConfSetting<ui64, false> _TableTimeout; + NCommon::TConfSetting<ui64, false> _LongWorkersAllocationWarnTimeout; + NCommon::TConfSetting<ui64, false> _LongWorkersAllocationFailTimeout; NCommon::TConfSetting<bool, false> EnableInsert; - NCommon::TConfSetting<ui64, false> _AllResultsBytesLimit; - NCommon::TConfSetting<ui64, false> _RowsLimitPerWrite; - NCommon::TConfSetting<bool, false> EnableStrip; - NCommon::TConfSetting<bool, false> EnableComputeActor; - NCommon::TConfSetting<TString, false> ComputeActorType; - NCommon::TConfSetting<bool, false> EnablePorto; // Will be renamed to _EnablePorto - NCommon::TConfSetting<ui64, false> _PortoMemoryLimit; - NCommon::TConfSetting<bool, false> EnableFullResultWrite; + NCommon::TConfSetting<ui64, false> _AllResultsBytesLimit; + NCommon::TConfSetting<ui64, false> _RowsLimitPerWrite; + NCommon::TConfSetting<bool, false> EnableStrip; + NCommon::TConfSetting<bool, false> EnableComputeActor; + NCommon::TConfSetting<TString, false> ComputeActorType; + NCommon::TConfSetting<bool, false> EnablePorto; // Will be renamed to _EnablePorto + NCommon::TConfSetting<ui64, false> _PortoMemoryLimit; + NCommon::TConfSetting<bool, false> EnableFullResultWrite; NCommon::TConfSetting<bool, false> _OneGraphPerQuery; - NCommon::TConfSetting<TString, false> _FallbackOnRuntimeErrors; - - NCommon::TConfSetting<TString, false> WorkerFilter; - - // This options will be passed to executor_actor and worker_actor - template <typename TProtoConfig> - void Save(TProtoConfig& config) { -#define SAVE_SETTING(name) \ - if (this->name.Get()) { \ - auto* s = config.AddSettings(); \ - s->SetName(#name); \ - s->SetValue(ToString(*this->name.Get())); \ - } - + NCommon::TConfSetting<TString, false> _FallbackOnRuntimeErrors; + + NCommon::TConfSetting<TString, false> WorkerFilter; + + // This options will be passed to executor_actor and worker_actor + template <typename TProtoConfig> + void Save(TProtoConfig& config) { +#define SAVE_SETTING(name) \ + if (this->name.Get()) { \ + auto* s = config.AddSettings(); \ + s->SetName(#name); \ + s->SetValue(ToString(*this->name.Get())); \ + } + SAVE_SETTING(AnalyticsHopping); - SAVE_SETTING(MaxRetries); - SAVE_SETTING(MaxNetworkRetries); - SAVE_SETTING(WorkersPerOperation); - SAVE_SETTING(RetryBackoffMs); - SAVE_SETTING(FallbackPolicy); - SAVE_SETTING(CollectCoreDumps); - SAVE_SETTING(PullRequestTimeoutMs); - SAVE_SETTING(PingTimeoutMs); + SAVE_SETTING(MaxRetries); + SAVE_SETTING(MaxNetworkRetries); + SAVE_SETTING(WorkersPerOperation); + SAVE_SETTING(RetryBackoffMs); + SAVE_SETTING(FallbackPolicy); + SAVE_SETTING(CollectCoreDumps); + SAVE_SETTING(PullRequestTimeoutMs); + SAVE_SETTING(PingTimeoutMs); SAVE_SETTING(OptLLVM); - SAVE_SETTING(ChannelBufferSize); - SAVE_SETTING(OutputChunkMaxSize); + SAVE_SETTING(ChannelBufferSize); + SAVE_SETTING(OutputChunkMaxSize); SAVE_SETTING(MemoryLimit); - SAVE_SETTING(_LiteralTimeout); - SAVE_SETTING(_TableTimeout); - SAVE_SETTING(_LongWorkersAllocationWarnTimeout); - SAVE_SETTING(_LongWorkersAllocationFailTimeout); - SAVE_SETTING(_AllResultsBytesLimit); - SAVE_SETTING(_RowsLimitPerWrite); - SAVE_SETTING(EnableComputeActor); - SAVE_SETTING(EnablePorto); - SAVE_SETTING(_PortoMemoryLimit); - SAVE_SETTING(EnableFullResultWrite); - SAVE_SETTING(_FallbackOnRuntimeErrors); - SAVE_SETTING(WorkerFilter); - SAVE_SETTING(ComputeActorType); - -#undef SAVE_SETTING - } - - TDqSettings::TPtr WithFillSettings(const IDataProvider::TFillSettings& fillSettings) const { + SAVE_SETTING(_LiteralTimeout); + SAVE_SETTING(_TableTimeout); + SAVE_SETTING(_LongWorkersAllocationWarnTimeout); + SAVE_SETTING(_LongWorkersAllocationFailTimeout); + SAVE_SETTING(_AllResultsBytesLimit); + SAVE_SETTING(_RowsLimitPerWrite); + SAVE_SETTING(EnableComputeActor); + SAVE_SETTING(EnablePorto); + SAVE_SETTING(_PortoMemoryLimit); + SAVE_SETTING(EnableFullResultWrite); + SAVE_SETTING(_FallbackOnRuntimeErrors); + SAVE_SETTING(WorkerFilter); + SAVE_SETTING(ComputeActorType); + +#undef SAVE_SETTING + } + + TDqSettings::TPtr WithFillSettings(const IDataProvider::TFillSettings& fillSettings) const { auto copy = std::make_shared<TDqSettings>(*this); - if (fillSettings.RowsLimitPerWrite) { - copy->_RowsLimitPerWrite = *fillSettings.RowsLimitPerWrite; - } - if (fillSettings.AllResultsBytesLimit) { - copy->_AllResultsBytesLimit = *fillSettings.AllResultsBytesLimit; - } - - return copy; - } -}; - -struct TDqConfiguration: public TDqSettings, public NCommon::TSettingDispatcher { - using TPtr = TIntrusivePtr<TDqConfiguration>; - - TDqConfiguration(); - TDqConfiguration(const TDqConfiguration&) = delete; - - template <class TProtoConfig> - void Init(const TProtoConfig& config, const TString& userName) - { - // Init settings from config + if (fillSettings.RowsLimitPerWrite) { + copy->_RowsLimitPerWrite = *fillSettings.RowsLimitPerWrite; + } + if (fillSettings.AllResultsBytesLimit) { + copy->_AllResultsBytesLimit = *fillSettings.AllResultsBytesLimit; + } + + return copy; + } +}; + +struct TDqConfiguration: public TDqSettings, public NCommon::TSettingDispatcher { + using TPtr = TIntrusivePtr<TDqConfiguration>; + + TDqConfiguration(); + TDqConfiguration(const TDqConfiguration&) = delete; + + template <class TProtoConfig> + void Init(const TProtoConfig& config, const TString& userName) + { + // Init settings from config this->Dispatch(config.GetDefaultSettings(), userName); - + // TODO: drop after releasing new gateways config - if (this->AnalyzeQuery.Get().Empty()) { - int percent = 0; - - if ((percent = this->_AnalyzeQueryPercentage.Get().GetOrElse(0)) && - RandomNumber<ui8>(100) < percent) - { - this->Dispatch(NCommon::ALL_CLUSTERS, "AnalyzeQuery", "true", TDqConfiguration::EStage::STATIC); - } - - for (const auto& userFromConfig : config.GetDefaultAnalyzeQueryForUsers()) { - if (userFromConfig == userName) { - this->Dispatch(NCommon::ALL_CLUSTERS, "AnalyzeQuery", "true", TDqConfiguration::EStage::STATIC); - break; - } - } - } - - this->FreezeDefaults(); - } -}; - -} //namespace NYql + if (this->AnalyzeQuery.Get().Empty()) { + int percent = 0; + + if ((percent = this->_AnalyzeQueryPercentage.Get().GetOrElse(0)) && + RandomNumber<ui8>(100) < percent) + { + this->Dispatch(NCommon::ALL_CLUSTERS, "AnalyzeQuery", "true", TDqConfiguration::EStage::STATIC); + } + + for (const auto& userFromConfig : config.GetDefaultAnalyzeQueryForUsers()) { + if (userFromConfig == userName) { + this->Dispatch(NCommon::ALL_CLUSTERS, "AnalyzeQuery", "true", TDqConfiguration::EStage::STATIC); + break; + } + } + } + + this->FreezeDefaults(); + } +}; + +} //namespace NYql diff --git a/ydb/library/yql/providers/dq/config/config.proto b/ydb/library/yql/providers/dq/config/config.proto index 54de861993..1cae395e7e 100644 --- a/ydb/library/yql/providers/dq/config/config.proto +++ b/ydb/library/yql/providers/dq/config/config.proto @@ -92,7 +92,7 @@ message TDqConfig { optional TPortoSettings PortoSettings = 31; optional bool ContainerCpuLimit = 26; // for testing only optional TSolomon Solomon = 30; - optional bool CanUseComputeActor = 32 [default = false]; + optional bool CanUseComputeActor = 32 [default = false]; } repeated TYtBackend YtBackends = 5; diff --git a/ydb/library/yql/providers/dq/counters/counters.h b/ydb/library/yql/providers/dq/counters/counters.h index c9289f906f..4a457d24fc 100644 --- a/ydb/library/yql/providers/dq/counters/counters.h +++ b/ydb/library/yql/providers/dq/counters/counters.h @@ -1,65 +1,65 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_execution.h> #include <ydb/library/yql/dq/runtime/dq_input_channel.h> #include <ydb/library/yql/dq/runtime/dq_output_channel.h> #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> - -#include <util/string/split.h> - -namespace NYql { - -struct TCounters { - - static TString GetCounterName(const TString& prefix, const std::map<TString, TString>& labels, const TString& name) { - TStringBuilder counterName; - counterName << prefix << ":"; - for (const auto& [k, v] : labels) { - counterName << k << "=" << v << ","; - } - counterName << "Name=" << name; - return counterName; - } - - void CopyCounters(TOperationStatistics& statistics) const { - for (const auto& [k, v] : Counters) { - TOperationStatistics::TEntry entry( - k, - v.Sum, - v.Max, - v.Min, - v.Avg, - v.Count - ); - statistics.Entries.push_back(entry); - } - } - - template<typename T> - void CopyCounters(T& t) const { - for (const auto& [k, v] : Counters) { - auto* metric = t.AddMetric(); - metric->SetName(k); - metric->SetSum(v.Sum); - metric->SetMax(v.Max); - metric->SetMin(v.Min); - metric->SetAvg(v.Avg); - metric->SetCount(v.Count); - } - } - - template<typename T> - void FlushCounters(T& t) const { - CopyCounters(t); - Counters.clear(); - } - - template<typename T> - void AddCounter(const TString& name, T value) const { - auto& counter = Counters[name]; - counter.Count += value; - } - + +#include <util/string/split.h> + +namespace NYql { + +struct TCounters { + + static TString GetCounterName(const TString& prefix, const std::map<TString, TString>& labels, const TString& name) { + TStringBuilder counterName; + counterName << prefix << ":"; + for (const auto& [k, v] : labels) { + counterName << k << "=" << v << ","; + } + counterName << "Name=" << name; + return counterName; + } + + void CopyCounters(TOperationStatistics& statistics) const { + for (const auto& [k, v] : Counters) { + TOperationStatistics::TEntry entry( + k, + v.Sum, + v.Max, + v.Min, + v.Avg, + v.Count + ); + statistics.Entries.push_back(entry); + } + } + + template<typename T> + void CopyCounters(T& t) const { + for (const auto& [k, v] : Counters) { + auto* metric = t.AddMetric(); + metric->SetName(k); + metric->SetSum(v.Sum); + metric->SetMax(v.Max); + metric->SetMin(v.Min); + metric->SetAvg(v.Avg); + metric->SetCount(v.Count); + } + } + + template<typename T> + void FlushCounters(T& t) const { + CopyCounters(t); + Counters.clear(); + } + + template<typename T> + void AddCounter(const TString& name, T value) const { + auto& counter = Counters[name]; + counter.Count += value; + } + template<typename T> void SetCounter(const TString& name, T value) const { auto& counter = Counters[name]; @@ -70,160 +70,160 @@ struct TCounters { return Histograms[name]; } - void AddTimeCounter(const TString& name, i64 value) const { - AddCounter(name, TDuration::MilliSeconds(value)); - } - - void AddCounter(const TString& name, TDuration value) const { - auto val = value.MilliSeconds(); - auto& counter = Counters[name]; - counter.Sum += val; - counter.Min = counter.Count == 0 - ? val - : Min<i64>(counter.Min, val); - counter.Max = counter.Count == 0 - ? val - : Max<i64>(counter.Max, val); - counter.Avg = counter.Sum / (counter.Count + 1); - counter.Count += 1; - } - - template<typename T> - void AddCounters(const T& t) const { - for (const auto& m : t.GetMetric()) { - TEntry value; - value.Sum = m.GetSum(); - value.Max = m.GetMax(); - value.Min = m.GetMin(); - value.Avg = m.GetAvg(); - value.Count = m.GetCount(); - AddCounter(m.GetName(), value); - } - } - - template<typename T> - void AddCounters2(const T& t) const { - for (const auto& m : t) { - TEntry value; - value.Sum = m.Sum; - value.Max = m.Max; - value.Min = m.Min; - value.Avg = m.Avg; - value.Count = m.Count; - AddCounter(m.Name, value); - } - } - - void StartCounter(const TString& name, TInstant now = TInstant::Now()) const { - if (!Start.contains(name)) { - Start[name] = now; - } - } - - void FlushCounter(const TString& name) const { - auto it = Start.find(name); - if (it != Start.end()) { - AddCounter(name, TInstant::Now() - it->second); - Start.erase(it); - } - } - - const auto& Get() const { - return Counters; - } - + void AddTimeCounter(const TString& name, i64 value) const { + AddCounter(name, TDuration::MilliSeconds(value)); + } + + void AddCounter(const TString& name, TDuration value) const { + auto val = value.MilliSeconds(); + auto& counter = Counters[name]; + counter.Sum += val; + counter.Min = counter.Count == 0 + ? val + : Min<i64>(counter.Min, val); + counter.Max = counter.Count == 0 + ? val + : Max<i64>(counter.Max, val); + counter.Avg = counter.Sum / (counter.Count + 1); + counter.Count += 1; + } + + template<typename T> + void AddCounters(const T& t) const { + for (const auto& m : t.GetMetric()) { + TEntry value; + value.Sum = m.GetSum(); + value.Max = m.GetMax(); + value.Min = m.GetMin(); + value.Avg = m.GetAvg(); + value.Count = m.GetCount(); + AddCounter(m.GetName(), value); + } + } + + template<typename T> + void AddCounters2(const T& t) const { + for (const auto& m : t) { + TEntry value; + value.Sum = m.Sum; + value.Max = m.Max; + value.Min = m.Min; + value.Avg = m.Avg; + value.Count = m.Count; + AddCounter(m.Name, value); + } + } + + void StartCounter(const TString& name, TInstant now = TInstant::Now()) const { + if (!Start.contains(name)) { + Start[name] = now; + } + } + + void FlushCounter(const TString& name) const { + auto it = Start.find(name); + if (it != Start.end()) { + AddCounter(name, TInstant::Now() - it->second); + Start.erase(it); + } + } + + const auto& Get() const { + return Counters; + } + const auto& GetHistograms() const { return Histograms; } - struct TEntry { - i64 Sum = 0; - i64 Max = 0; - i64 Min = 0; - i64 Avg = 0; - i64 Count = 0; - }; - - struct TCounterBlock { - TCounterBlock(const TCounters* parent, const TString& name) - : Name(name) - , StartTime(TInstant::Now()) - , Parent(parent) - { } - - ~TCounterBlock() { - Parent->AddCounter(Name, TInstant::Now() - StartTime); - } - - const TString Name; - const TInstant StartTime; - const TCounters* Parent; - }; - - std::unique_ptr<TCounterBlock> MeasureBlock(const TString& name) const { - return std::make_unique<TCounterBlock>(this, name); - } - - template<typename T> - T Measure(const TString& name, const std::function<T(void)>& f) const { - MeasureBlock(name); - return f(); - } - - void AddCounter(const TString& name, const TEntry& value) const { - auto& counter = Counters[name]; - if (value.Count) { - counter.Sum += value.Sum; - counter.Min = counter.Count == 0 - ? value.Min - : Min(counter.Min, value.Min); - counter.Max = counter.Count == 0 - ? value.Max - : Max(counter.Max, value.Max); - counter.Count += value.Count; - counter.Avg = counter.Sum / counter.Count; - } - } - - void Clear() const { - Counters.clear(); - Start.clear(); - } - -#define ADD_COUNTER(name) \ - do { \ - auto value = currentStats.name - oldStats.name; \ - if (value) { \ - AddCounter(GetCounterName("TaskRunner", labels, #name), value); \ - } \ - oldStats.name = currentStats.name; \ - } while (0); - - void AddInputChannelStats( - const NDq::TDqInputChannelStats& currentStats, - NDq::TDqInputChannelStats& oldStats, - ui64 taskId, - ui64 channelId) - { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)}, - {"InputChannel", ToString(channelId)} - }; - - ADD_COUNTER(Chunks); - ADD_COUNTER(Bytes); - ADD_COUNTER(RowsIn); - ADD_COUNTER(RowsOut); - ADD_COUNTER(RowsInMemory); - ADD_COUNTER(MaxMemoryUsage); - ADD_COUNTER(DeserializationTime); - } - - void AddSourceStats( - const NDq::TDqSourceStats& currentStats, - NDq::TDqSourceStats& oldStats, - ui64 taskId, ui64 inputIndex) - { + struct TEntry { + i64 Sum = 0; + i64 Max = 0; + i64 Min = 0; + i64 Avg = 0; + i64 Count = 0; + }; + + struct TCounterBlock { + TCounterBlock(const TCounters* parent, const TString& name) + : Name(name) + , StartTime(TInstant::Now()) + , Parent(parent) + { } + + ~TCounterBlock() { + Parent->AddCounter(Name, TInstant::Now() - StartTime); + } + + const TString Name; + const TInstant StartTime; + const TCounters* Parent; + }; + + std::unique_ptr<TCounterBlock> MeasureBlock(const TString& name) const { + return std::make_unique<TCounterBlock>(this, name); + } + + template<typename T> + T Measure(const TString& name, const std::function<T(void)>& f) const { + MeasureBlock(name); + return f(); + } + + void AddCounter(const TString& name, const TEntry& value) const { + auto& counter = Counters[name]; + if (value.Count) { + counter.Sum += value.Sum; + counter.Min = counter.Count == 0 + ? value.Min + : Min(counter.Min, value.Min); + counter.Max = counter.Count == 0 + ? value.Max + : Max(counter.Max, value.Max); + counter.Count += value.Count; + counter.Avg = counter.Sum / counter.Count; + } + } + + void Clear() const { + Counters.clear(); + Start.clear(); + } + +#define ADD_COUNTER(name) \ + do { \ + auto value = currentStats.name - oldStats.name; \ + if (value) { \ + AddCounter(GetCounterName("TaskRunner", labels, #name), value); \ + } \ + oldStats.name = currentStats.name; \ + } while (0); + + void AddInputChannelStats( + const NDq::TDqInputChannelStats& currentStats, + NDq::TDqInputChannelStats& oldStats, + ui64 taskId, + ui64 channelId) + { + std::map<TString, TString> labels = { + {"Task", ToString(taskId)}, + {"InputChannel", ToString(channelId)} + }; + + ADD_COUNTER(Chunks); + ADD_COUNTER(Bytes); + ADD_COUNTER(RowsIn); + ADD_COUNTER(RowsOut); + ADD_COUNTER(RowsInMemory); + ADD_COUNTER(MaxMemoryUsage); + ADD_COUNTER(DeserializationTime); + } + + void AddSourceStats( + const NDq::TDqSourceStats& currentStats, + NDq::TDqSourceStats& oldStats, + ui64 taskId, ui64 inputIndex) + { std::map<TString, TString> labels = { {"Task", ToString(taskId)}, {"SourceIndex", ToString(inputIndex)} @@ -238,57 +238,57 @@ struct TCounters { ADD_COUNTER(InputIndex); } - void AddOutputChannelStats( - const NDq::TDqOutputChannelStats& currentStats, - NDq::TDqOutputChannelStats& oldStats, - ui64 taskId, ui64 channelId) - { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)}, - {"OutputChannel", ToString(channelId)} - }; - - ADD_COUNTER(Chunks) - ADD_COUNTER(Bytes); - ADD_COUNTER(RowsIn); - ADD_COUNTER(RowsOut); - ADD_COUNTER(MaxMemoryUsage); - ADD_COUNTER(MaxRowsInMemory); - - ADD_COUNTER(SerializationTime); - - ADD_COUNTER(SpilledBytes); - ADD_COUNTER(SpilledRows); - ADD_COUNTER(SpilledBlobs); - } - - void AddTaskRunnerStats( - const NDq::TDqTaskRunnerStats& currentStats, - NDq::TDqTaskRunnerStats& oldStats, - ui64 taskId) - { - std::map<TString, TString> labels = { - {"Task", ToString(taskId)} - }; - - // basic stats + void AddOutputChannelStats( + const NDq::TDqOutputChannelStats& currentStats, + NDq::TDqOutputChannelStats& oldStats, + ui64 taskId, ui64 channelId) + { + std::map<TString, TString> labels = { + {"Task", ToString(taskId)}, + {"OutputChannel", ToString(channelId)} + }; + + ADD_COUNTER(Chunks) + ADD_COUNTER(Bytes); + ADD_COUNTER(RowsIn); + ADD_COUNTER(RowsOut); + ADD_COUNTER(MaxMemoryUsage); + ADD_COUNTER(MaxRowsInMemory); + + ADD_COUNTER(SerializationTime); + + ADD_COUNTER(SpilledBytes); + ADD_COUNTER(SpilledRows); + ADD_COUNTER(SpilledBlobs); + } + + void AddTaskRunnerStats( + const NDq::TDqTaskRunnerStats& currentStats, + NDq::TDqTaskRunnerStats& oldStats, + ui64 taskId) + { + std::map<TString, TString> labels = { + {"Task", ToString(taskId)} + }; + + // basic stats ADD_COUNTER(ComputeCpuTime) ADD_COUNTER(BuildCpuTime) - - // profile stats - ADD_COUNTER(WaitTime) - ADD_COUNTER(WaitOutputTime) - } - -#undef ADD_COUNTER - -protected: - - mutable THashMap<TString, TEntry> Counters; + + // profile stats + ADD_COUNTER(WaitTime) + ADD_COUNTER(WaitOutputTime) + } + +#undef ADD_COUNTER + +protected: + + mutable THashMap<TString, TEntry> Counters; mutable THashMap<TString, THashMap<i64, ui64>> Histograms; - mutable THashMap<TString, TInstant> Start; -}; - + mutable THashMap<TString, TInstant> Start; +}; + TCounters AggregateQueryStatsByStage(TCounters& queryStat, const THashMap<ui64, ui64>& task2Stage); -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json b/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json index 335fe4560c..566f8d0503 100644 --- a/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json +++ b/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json @@ -58,4 +58,4 @@ "Match": {"Type": "Callable", "Name": "DqSourceWideWrap"} } ] -} +} diff --git a/ydb/library/yql/providers/dq/expr_nodes/ya.make b/ydb/library/yql/providers/dq/expr_nodes/ya.make index 361e711568..0d6b0d4f7b 100644 --- a/ydb/library/yql/providers/dq/expr_nodes/ya.make +++ b/ydb/library/yql/providers/dq/expr_nodes/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER( - g:yql + g:yql ) SRCS( @@ -33,4 +33,4 @@ RUN_PROGRAM( ${ARCADIA_ROOT}/util/generic/hash_set.h ) -END() +END() diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_integration.h b/ydb/library/yql/providers/dq/interface/yql_dq_integration.h index aa59730112..ee1938e698 100644 --- a/ydb/library/yql/providers/dq/interface/yql_dq_integration.h +++ b/ydb/library/yql/providers/dq/interface/yql_dq_integration.h @@ -13,7 +13,7 @@ namespace NYql { -struct TDqSettings; +struct TDqSettings; namespace NCommon { class TMkqlCallableCompilerBase; @@ -32,7 +32,7 @@ public: virtual TExprNode::TPtr WrapRead(const TDqSettings& config, const TExprNode::TPtr& read, TExprContext& ctx) = 0; virtual TMaybe<bool> CanWrite(const TDqSettings& config, const TExprNode& write, TExprContext& ctx) = 0; virtual void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) = 0; - virtual bool CanFallback() = 0; + virtual bool CanFallback() = 0; virtual void FillSourceSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sourceType) = 0; virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0; virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0; diff --git a/ydb/library/yql/providers/dq/local_gateway/ya.make b/ydb/library/yql/providers/dq/local_gateway/ya.make index 8ca82bb8dd..aa1ed08342 100644 --- a/ydb/library/yql/providers/dq/local_gateway/ya.make +++ b/ydb/library/yql/providers/dq/local_gateway/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - -OWNER(g:yql) - +LIBRARY() + +OWNER(g:yql) + YQL_LAST_ABI_VERSION() -SRCS( - yql_dq_gateway_local.cpp -) - -PEERDIR( +SRCS( + yql_dq_gateway_local.cpp +) + +PEERDIR( ydb/library/yql/utils ydb/library/yql/dq/actors/compute ydb/library/yql/providers/dq/provider ydb/library/yql/providers/dq/api/protos ydb/library/yql/providers/dq/task_runner - ydb/library/yql/providers/dq/worker_manager + ydb/library/yql/providers/dq/worker_manager ydb/library/yql/providers/pq/async_io ydb/library/yql/providers/s3/actors ydb/library/yql/providers/ydb/actors ydb/library/yql/providers/clickhouse/actors - ydb/library/yql/providers/dq/service -) - -END() + ydb/library/yql/providers/dq/service +) + +END() diff --git a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp index d5cfd2aa73..1a8d3172e4 100644 --- a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp +++ b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp @@ -1,29 +1,29 @@ -#include "yql_dq_gateway_local.h" - +#include "yql_dq_gateway_local.h" + #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h> - -#include <ydb/library/yql/providers/dq/service/interconnect_helpers.h> -#include <ydb/library/yql/providers/dq/service/service_node.h> - + +#include <ydb/library/yql/providers/dq/service/interconnect_helpers.h> +#include <ydb/library/yql/providers/dq/service/service_node.h> + #include <ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h> #include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h> #include <ydb/library/yql/providers/clickhouse/actors/yql_ch_source_factory.h> #include <ydb/library/yql/providers/s3/actors/yql_s3_source_factory.h> #include <ydb/library/yql/providers/ydb/actors/yql_ydb_source_factory.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_io_actors_factory.h> #include <ydb/library/yql/utils/range_walker.h> #include <ydb/library/yql/utils/bind_in_range.h> - -#include <library/cpp/messagebus/network.h> - -namespace NYql { - -using namespace NActors; -using NDqs::MakeWorkerManagerActorID; - + +#include <library/cpp/messagebus/network.h> + +namespace NYql { + +using namespace NActors; +using NDqs::MakeWorkerManagerActorID; + namespace { // TODO: Use the only driver for both sources. NDq::IDqSourceActorFactory::TPtr CreateSourceActorFactory(const NYdb::TDriver& driver, IHTTPGateway::TPtr httpGateway) { @@ -42,115 +42,115 @@ namespace { } } -class TLocalServiceHolder { -public: +class TLocalServiceHolder { +public: TLocalServiceHolder(NYdb::TDriver driver, IHTTPGateway::TPtr httpGateway, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort) - { - ui32 nodeId = 1; - - TString hostName; - TString localAddress; - std::tie(hostName, localAddress) = NDqs::GetLocalAddress(); - - NDqs::TServiceNodeConfig config = { - nodeId, - localAddress, - hostName, - static_cast<ui16>(interconnectPort.Addr.GetPort()), - static_cast<ui16>(grpcPort.Addr.GetPort()), - 0, // mbus - interconnectPort.Socket.Get()->Release(), - grpcPort.Socket.Get()->Release(), - }; - - ServiceNode = MakeHolder<TServiceNode>( - config, - 1, - CreateMetricsRegistry(GetSensorsGroupFor(NSensorComponent::kDq))); - - NDqs::TLocalWorkerManagerOptions lwmOptions; - lwmOptions.Factory = NTaskRunnerProxy::CreateFactory(functionRegistry, compFactory, taskTransformFactory, true); - lwmOptions.SourceActorFactory = CreateSourceActorFactory(driver, std::move(httpGateway)); - lwmOptions.SinkActorFactory = CreateSinkActorFactory(driver); - lwmOptions.TaskRunnerInvokerFactory = new NDqs::TTaskRunnerInvokerFactory(); - lwmOptions.TaskRunnerActorFactory = NDq::NTaskRunnerActor::CreateLocalTaskRunnerActorFactory( - [=](const NDqProto::TDqTask& task, const NDq::TLogFunc& ) - { - return lwmOptions.Factory->Get(task); - }); - auto resman = NDqs::CreateLocalWorkerManager(lwmOptions); - - ServiceNode->AddLocalService( - MakeWorkerManagerActorID(nodeId), - TActorSetupCmd(resman, TMailboxType::Simple, 0)); - - ServiceNode->StartActorSystem(); + { + ui32 nodeId = 1; + + TString hostName; + TString localAddress; + std::tie(hostName, localAddress) = NDqs::GetLocalAddress(); + + NDqs::TServiceNodeConfig config = { + nodeId, + localAddress, + hostName, + static_cast<ui16>(interconnectPort.Addr.GetPort()), + static_cast<ui16>(grpcPort.Addr.GetPort()), + 0, // mbus + interconnectPort.Socket.Get()->Release(), + grpcPort.Socket.Get()->Release(), + }; + + ServiceNode = MakeHolder<TServiceNode>( + config, + 1, + CreateMetricsRegistry(GetSensorsGroupFor(NSensorComponent::kDq))); + + NDqs::TLocalWorkerManagerOptions lwmOptions; + lwmOptions.Factory = NTaskRunnerProxy::CreateFactory(functionRegistry, compFactory, taskTransformFactory, true); + lwmOptions.SourceActorFactory = CreateSourceActorFactory(driver, std::move(httpGateway)); + lwmOptions.SinkActorFactory = CreateSinkActorFactory(driver); + lwmOptions.TaskRunnerInvokerFactory = new NDqs::TTaskRunnerInvokerFactory(); + lwmOptions.TaskRunnerActorFactory = NDq::NTaskRunnerActor::CreateLocalTaskRunnerActorFactory( + [=](const NDqProto::TDqTask& task, const NDq::TLogFunc& ) + { + return lwmOptions.Factory->Get(task); + }); + auto resman = NDqs::CreateLocalWorkerManager(lwmOptions); + + ServiceNode->AddLocalService( + MakeWorkerManagerActorID(nodeId), + TActorSetupCmd(resman, TMailboxType::Simple, 0)); + + ServiceNode->StartActorSystem(); ServiceNode->StartService(dqTaskPreprocessorFactories); - } - - ~TLocalServiceHolder() - { - ServiceNode->Stop(); - } - -private: - THolder<TServiceNode> ServiceNode; -}; - -class TDqGatewayLocal: public IDqGateway -{ -public: - TDqGatewayLocal(THolder<TLocalServiceHolder>&& localService, const IDqGateway::TPtr& gateway) - : LocalService(std::move(localService)) - , Gateway(gateway) - { } - + } + + ~TLocalServiceHolder() + { + ServiceNode->Stop(); + } + +private: + THolder<TServiceNode> ServiceNode; +}; + +class TDqGatewayLocal: public IDqGateway +{ +public: + TDqGatewayLocal(THolder<TLocalServiceHolder>&& localService, const IDqGateway::TPtr& gateway) + : LocalService(std::move(localService)) + , Gateway(gateway) + { } + NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override { return Gateway->OpenSession(sessionId, username); - } - - void CloseSession(const TString& sessionId) override { + } + + void CloseSession(const TString& sessionId) override { return Gateway->CloseSession(sessionId); - } - + } + NThreading::TFuture<TResult> ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns, const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams, const TDqSettings::TPtr& settings, const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping, bool discard) override - { + { return Gateway->ExecutePlan(sessionId, plan, columns, secureParams, graphParams, settings, progressWriter, modulesMapping, discard); - } - -private: - THolder<TLocalServiceHolder> LocalService; - IDqGateway::TPtr Gateway; -}; - + } + +private: + THolder<TLocalServiceHolder> LocalService; + IDqGateway::TPtr Gateway; +}; + THolder<TLocalServiceHolder> CreateLocalServiceHolder(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, IHTTPGateway::TPtr gateway, NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort) { return MakeHolder<TLocalServiceHolder>(driver, std::move(gateway), functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, interconnectPort, grpcPort); -} - +} + TIntrusivePtr<IDqGateway> CreateLocalDqGateway(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, IHTTPGateway::TPtr gateway) { - int startPort = 31337; - TRangeWalker<int> portWalker(startPort, startPort+100); - auto interconnectPort = BindInRange(portWalker)[1]; - auto grpcPort = BindInRange(portWalker)[1]; - - return new TDqGatewayLocal( + int startPort = 31337; + TRangeWalker<int> portWalker(startPort, startPort+100); + auto interconnectPort = BindInRange(portWalker)[1]; + auto grpcPort = BindInRange(portWalker)[1]; + + return new TDqGatewayLocal( CreateLocalServiceHolder(driver, functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, std::move(gateway), interconnectPort, grpcPort), CreateDqGateway(std::get<0>(NDqs::GetLocalAddress()), grpcPort.Addr.GetPort(), 8)); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h index b5f5419d65..b248e7c833 100644 --- a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h +++ b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h> - -namespace NYql { - -TIntrusivePtr<IDqGateway> CreateLocalDqGateway(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - NKikimr::NMiniKQL::TComputationNodeFactory compFactory, + +namespace NYql { + +TIntrusivePtr<IDqGateway> CreateLocalDqGateway(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, IHTTPGateway::TPtr gateway = {}); - -} // namespace NYql + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp index 4ff33afcf7..b4bc07fbef 100644 --- a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp +++ b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp @@ -6,7 +6,7 @@ #include <ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h> #include <ydb/library/yql/core/type_ann/type_ann_core.h> #include <ydb/library/yql/core/yql_expr_type_annotation.h> - + #include <ydb/library/yql/dq/opt/dq_opt.h> #include <ydb/library/yql/dq/opt/dq_opt_phy.h> #include <ydb/library/yql/dq/opt/dq_opt_phy_finalizing.h> @@ -15,7 +15,7 @@ #include <ydb/library/yql/dq/type_ann/dq_type_ann.h> #include <ydb/library/yql/utils/log/log.h> - + #include <util/string/split.h> #define PERFORM_RULE(func, ...) \ @@ -53,7 +53,7 @@ namespace NYql::NDqs { .List(input) .Build() .Build() - .Settings(TDqStageSettings().BuildNode(ctx, input->Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, input->Pos())) .Build() .Index() .Build("0") @@ -87,7 +87,7 @@ namespace NYql::NDqs { .Args({"row"}) .Body("row") .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Build() .Index() .Build("0") @@ -117,7 +117,7 @@ namespace NYql::NDqs { } namespace NPeephole { - + class TDqsPeepholeTransformer: public TSyncTransformerBase { public: TDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer, @@ -134,22 +134,22 @@ namespace NYql::NDqs { } auto transformer = CreateDqsRewritePhyCallablesTransformer(); - auto status = InstantTransform(*transformer, inputExpr, ctx); - if (status.Level != TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(inputExpr->Pos()), TString("Peephole optimization failed for Dq stage"))); - return TStatus::Error; - } - - bool hasNonDeterministicFunctions = false; - status = PeepHoleOptimizeNode<true>(inputExpr, inputExpr, ctx, TypesCtx, TypeAnnTransformer.Get(), hasNonDeterministicFunctions); - if (status.Level != TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(inputExpr->Pos()), TString("Peephole optimization failed for Dq stage"))); - return TStatus::Error; - } - - outputExpr = inputExpr; - Optimized = true; - return TStatus::Ok; + auto status = InstantTransform(*transformer, inputExpr, ctx); + if (status.Level != TStatus::Ok) { + ctx.AddError(TIssue(ctx.GetPosition(inputExpr->Pos()), TString("Peephole optimization failed for Dq stage"))); + return TStatus::Error; + } + + bool hasNonDeterministicFunctions = false; + status = PeepHoleOptimizeNode<true>(inputExpr, inputExpr, ctx, TypesCtx, TypeAnnTransformer.Get(), hasNonDeterministicFunctions); + if (status.Level != TStatus::Ok) { + ctx.AddError(TIssue(ctx.GetPosition(inputExpr->Pos()), TString("Peephole optimization failed for Dq stage"))); + return TStatus::Error; + } + + outputExpr = inputExpr; + Optimized = true; + return TStatus::Ok; } void Rewind() final { diff --git a/ydb/library/yql/providers/dq/opt/dqs_opt.h b/ydb/library/yql/providers/dq/opt/dqs_opt.h index 4768a0fea9..6c9e1a7c18 100644 --- a/ydb/library/yql/providers/dq/opt/dqs_opt.h +++ b/ydb/library/yql/providers/dq/opt/dqs_opt.h @@ -12,5 +12,5 @@ namespace NYql::NDqs { THolder<IGraphTransformer> CreateDqsBuildTransformer(); THolder<IGraphTransformer> CreateDqsRewritePhyCallablesTransformer(); THolder<IGraphTransformer> CreateDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer, TTypeAnnotationContext& typesCtx); - -} // namespace NYql::NDqs + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp index b0b9ff4a1d..9e87f81e7c 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp @@ -50,9 +50,9 @@ public: AddHandler(0, &TCoAggregate::Match, HNDL(RewriteAggregate)); AddHandler(0, &TCoTake::Match, HNDL(RewriteTakeSortToTopSort)); AddHandler(0, &TCoEquiJoin::Match, HNDL(RewriteEquiJoin)); - AddHandler(0, &TCoCalcOverWindow::Match, HNDL(ExpandWindowFunctions)); - AddHandler(0, &TCoCalcOverWindowGroup::Match, HNDL(ExpandWindowFunctions)); - AddHandler(0, &TCoFlatMapBase::Match, HNDL(FlatMapOverExtend)); + AddHandler(0, &TCoCalcOverWindow::Match, HNDL(ExpandWindowFunctions)); + AddHandler(0, &TCoCalcOverWindowGroup::Match, HNDL(ExpandWindowFunctions)); + AddHandler(0, &TCoFlatMapBase::Match, HNDL(FlatMapOverExtend)); AddHandler(0, &TDqQuery::Match, HNDL(MergeQueriesWithSinks)); AddHandler(0, &TDqStageBase::Match, HNDL(UnorderedInStage)); #undef HNDL @@ -68,10 +68,10 @@ protected: return node; } - TMaybeNode<TExprBase> FlatMapOverExtend(TExprBase node, TExprContext& ctx) { - return DqFlatMapOverExtend(node, ctx); - } - + TMaybeNode<TExprBase> FlatMapOverExtend(TExprBase node, TExprContext& ctx) { + return DqFlatMapOverExtend(node, ctx); + } + TMaybeNode<TExprBase> RewriteAggregate(TExprBase node, TExprContext& ctx) { auto aggregate = node.Cast<TCoAggregate>(); auto input = aggregate.Input().Maybe<TDqConnection>(); @@ -110,13 +110,13 @@ protected: } return DqRewriteEquiJoin(node, ctx); } - - TMaybeNode<TExprBase> ExpandWindowFunctions(TExprBase node, TExprContext& ctx) { + + TMaybeNode<TExprBase> ExpandWindowFunctions(TExprBase node, TExprContext& ctx) { if (node.Cast<TCoInputBase>().Input().Maybe<TDqConnection>()) { return DqExpandWindowFunctions(node, ctx, true); } return node; - } + } TMaybeNode<TExprBase> MergeQueriesWithSinks(TExprBase node, TExprContext& ctx) { return DqMergeQueriesWithSinks(node, ctx); diff --git a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp index 03f8791ca3..858e3da45a 100644 --- a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp @@ -34,7 +34,7 @@ public: AddHandler(0, &TCoSort::Match, HNDL(BuildSortStage<false>)); AddHandler(0, &TCoTake::Match, HNDL(BuildTakeOrTakeSkipStage<false>)); AddHandler(0, &TCoLength::Match, HNDL(RewriteLengthOfStageOutput)); - AddHandler(0, &TCoExtendBase::Match, HNDL(BuildExtendStage)); + AddHandler(0, &TCoExtendBase::Match, HNDL(BuildExtendStage)); AddHandler(0, &TDqJoin::Match, HNDL(RewriteRightJoinToLeft)); AddHandler(0, &TDqJoin::Match, HNDL(PushJoinToStage<false>)); AddHandler(0, &TDqJoin::Match, HNDL(BuildJoin<false>)); @@ -181,7 +181,7 @@ protected: .Stage<TDqStage>() .Inputs().Build() .Program(narrow) - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) .Build() .Index().Build("0") .Build() .Done(); @@ -245,12 +245,12 @@ protected: } template <bool IsGlobal> - TMaybeNode<TExprBase> BuildTakeOrTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) { - if (node.Maybe<TCoTake>().Input().Maybe<TCoSkip>()) { + TMaybeNode<TExprBase> BuildTakeOrTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) { + if (node.Maybe<TCoTake>().Input().Maybe<TCoSkip>()) { return DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal); - } else { + } else { return DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal); - } + } } TMaybeNode<TExprBase> RewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) { @@ -290,7 +290,7 @@ protected: if (!JoinPrerequisitesVerify(join, parentsMap, IsGlobal)) { return node; } - + return DqBuildJoinDict(join, ctx); // , optCtx); } diff --git a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h index d70560cdec..0b62dfce4a 100644 --- a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h +++ b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h @@ -17,7 +17,7 @@ namespace NYql::NDqs { struct TTaskMeta { THashMap<TString, TString> TaskParams; - TString ClusterNameHint; + TString ClusterNameHint; }; using TStageInfo = NYql::NDq::TStageInfo<TStageInfoMeta>; diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.cpp b/ydb/library/yql/providers/dq/planner/execution_planner.cpp index 37af97ebf2..ec404c0a6d 100644 --- a/ydb/library/yql/providers/dq/planner/execution_planner.cpp +++ b/ydb/library/yql/providers/dq/planner/execution_planner.cpp @@ -29,15 +29,15 @@ #include <stack> using namespace NYql; -using namespace NYql::NCommon; +using namespace NYql::NCommon; using namespace NYql::NDq; using namespace NYql::NDqProto; using namespace NYql::NNodes; using namespace NKikimr::NMiniKQL; -using namespace Yql::DqsProto; - +using namespace Yql::DqsProto; + namespace NYql::NDqs { namespace { TVector<TDqPhyStage> GetStages(const TExprNode::TPtr& exprRoot) { @@ -81,62 +81,62 @@ namespace NYql::NDqs { } } - TDqsExecutionPlanner::TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext, + TDqsExecutionPlanner::TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext, NYql::TExprContext& exprContext, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NYql::TExprNode::TPtr dqExprRoot, NActors::TActorId executerID, NActors::TActorId resultID) - : TypeContext(std::move(typeContext)) + : TypeContext(std::move(typeContext)) , ExprContext(exprContext) - , FunctionRegistry(functionRegistry) + , FunctionRegistry(functionRegistry) , DqExprRoot(std::move(dqExprRoot)) , ExecuterID(executerID) , ResultID(resultID) { } - void TDqsExecutionPlanner::Clear() { - TasksGraph.Clear(); - _MaxDataSizePerJob = 0; - } - - bool TDqsExecutionPlanner::CanFallback() { - THashSet<TString> sources; - bool flag = true; - - VisitExpr( + void TDqsExecutionPlanner::Clear() { + TasksGraph.Clear(); + _MaxDataSizePerJob = 0; + } + + bool TDqsExecutionPlanner::CanFallback() { + THashSet<TString> sources; + bool flag = true; + + VisitExpr( DqExprRoot, - [&](const TExprNode::TPtr& exprNode) { - if (exprNode->IsCallable("DataSource")) { - sources.insert(TString{exprNode->Child(0)->Content()}); - } - return true; - }); - - for (const auto& dataSourceName : sources) { - auto datasource = TypeContext->DataSourceMap.FindPtr(dataSourceName); - if (auto dqIntegration = (*datasource)->GetDqIntegration()) { - flag &= dqIntegration->CanFallback(); - } - } - - return flag; - } - - ui64 TDqsExecutionPlanner::StagesCount() { + [&](const TExprNode::TPtr& exprNode) { + if (exprNode->IsCallable("DataSource")) { + sources.insert(TString{exprNode->Child(0)->Content()}); + } + return true; + }); + + for (const auto& dataSourceName : sources) { + auto datasource = TypeContext->DataSourceMap.FindPtr(dataSourceName); + if (auto dqIntegration = (*datasource)->GetDqIntegration()) { + flag &= dqIntegration->CanFallback(); + } + } + + return flag; + } + + ui64 TDqsExecutionPlanner::StagesCount() { auto stages = GetStages(DqExprRoot); - return stages.size(); - } - + return stages.size(); + } + ui32 TDqsExecutionPlanner::PlanExecution(const TDqSettings::TPtr& settings, bool canFallback) { TExprBase expr(DqExprRoot); auto result = expr.Maybe<TDqCnResult>(); auto query = expr.Maybe<TDqQuery>(); - const auto maxTasksPerOperation = settings->MaxTasksPerOperation.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerOperation); - + const auto maxTasksPerOperation = settings->MaxTasksPerOperation.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerOperation); + YQL_LOG(DEBUG) << "Execution Plan " << NCommon::ExprToPrettyString(ExprContext, *DqExprRoot); - + auto stages = GetStages(DqExprRoot); YQL_ENSURE(!stages.empty()); @@ -147,9 +147,9 @@ namespace NYql::NDqs { for (const auto& stage : stages) { const bool hasDqSource = HasDqSource(stage); if ((hasDqSource || HasReadWraps(stage.Program().Ptr())) && BuildReadStage(settings, stage, hasDqSource, canFallback)) { - YQL_LOG(DEBUG) << "Read stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr()); + YQL_LOG(DEBUG) << "Read stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr()); } else { - YQL_LOG(DEBUG) << "Common stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr()); + YQL_LOG(DEBUG) << "Common stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr()); NDq::CommonBuildTasks(TasksGraph, stage); } @@ -184,17 +184,17 @@ namespace NYql::NDqs { } BuildConnections(stage); - + if (canFallback && TasksGraph.GetTasks().size() > maxTasksPerOperation) { - break; - } + break; + } + } + + for (const auto& stage : stages) { + auto& stageInfo = TasksGraph.GetStageInfo(stage); + YQL_ENSURE(!stageInfo.Tasks.empty()); } - for (const auto& stage : stages) { - auto& stageInfo = TasksGraph.GetStageInfo(stage); - YQL_ENSURE(!stageInfo.Tasks.empty()); - } - if (result) { auto& resultStageInfo = TasksGraph.GetStageInfo(result.Cast().Output().Stage().Cast<TDqPhyStage>()); YQL_ENSURE(resultStageInfo.Tasks.size() == 1); @@ -299,21 +299,21 @@ namespace NYql::NDqs { } } - // TODO: Split Build and Get stages - TVector<TDqTask>& TDqsExecutionPlanner::GetTasks() { - if (Tasks.empty()) { - auto workerCount = TasksGraph.GetTasks().size(); + // TODO: Split Build and Get stages + TVector<TDqTask>& TDqsExecutionPlanner::GetTasks() { + if (Tasks.empty()) { + auto workerCount = TasksGraph.GetTasks().size(); TVector<NActors::TActorId> workers; - for (unsigned int i = 0; i < workerCount; ++i) { + for (unsigned int i = 0; i < workerCount; ++i) { NActors::TActorId fakeActorId(i+1, 0, 0, 0); - workers.emplace_back(fakeActorId); - } - Tasks = GetTasks(workers); - } - return Tasks; - } - - TVector<TDqTask> TDqsExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) { + workers.emplace_back(fakeActorId); + } + Tasks = GetTasks(workers); + } + return Tasks; + } + + TVector<TDqTask> TDqsExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) { auto& tasks = TasksGraph.GetTasks(); YQL_ENSURE(tasks.size() == workers.size()); @@ -322,27 +322,27 @@ namespace NYql::NDqs { } THashMap<TStageId, std::tuple<TString, ui64>> stagePrograms = BuildAllPrograms(); - TVector<TDqTask> plan; - THashSet<TString> clusterNameHints; + TVector<TDqTask> plan; + THashSet<TString> clusterNameHints; for (const auto& task : tasks) { - if (task.Meta.ClusterNameHint) { - clusterNameHints.insert(task.Meta.ClusterNameHint); - } - } - for (const auto& task : tasks) { - Yql::DqsProto::TTaskMeta taskMeta; - TDqTask taskDesc; + if (task.Meta.ClusterNameHint) { + clusterNameHints.insert(task.Meta.ClusterNameHint); + } + } + for (const auto& task : tasks) { + Yql::DqsProto::TTaskMeta taskMeta; + TDqTask taskDesc; taskDesc.SetId(task.Id); NActors::ActorIdToProto(ExecuterID, taskDesc.MutableExecuter()->MutableActorId()); - auto& taskParams = *taskMeta.MutableTaskParams(); + auto& taskParams = *taskMeta.MutableTaskParams(); for (const auto& [k, v]: task.Meta.TaskParams) { taskParams[k] = v; } - if (clusterNameHints.size() == 1) { - taskMeta.SetClusterNameHint(*clusterNameHints.begin()); - } else { - taskMeta.SetClusterNameHint(task.Meta.ClusterNameHint); - } + if (clusterNameHints.size() == 1) { + taskMeta.SetClusterNameHint(*clusterNameHints.begin()); + } else { + taskMeta.SetClusterNameHint(task.Meta.ClusterNameHint); + } for (auto& input : task.Inputs) { auto& inputDesc = *taskDesc.AddInputs(); @@ -365,15 +365,15 @@ namespace NYql::NDqs { auto& program = *taskDesc.MutableProgram(); program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0); - TString programStr; - ui64 stageId; - std::tie(programStr, stageId) = stagePrograms[task.StageId]; - program.SetRaw(programStr); - taskMeta.SetStageId(stageId); - taskDesc.MutableMeta()->PackFrom(taskMeta); + TString programStr; + ui64 stageId; + std::tie(programStr, stageId) = stagePrograms[task.StageId]; + program.SetRaw(programStr); + taskMeta.SetStageId(stageId); + taskDesc.MutableMeta()->PackFrom(taskMeta); taskDesc.SetStageId(stageId); - plan.emplace_back(std::move(taskDesc)); + plan.emplace_back(std::move(taskDesc)); } return plan; @@ -387,7 +387,7 @@ namespace NYql::NDqs { } } - TString TDqsExecutionPlanner::GetResultType(bool withTagged) const { + TString TDqsExecutionPlanner::GetResultType(bool withTagged) const { if (SourceTaskID) { auto& stage = TasksGraph.GetStageInfo(TasksGraph.GetTask(SourceTaskID).StageId).Meta.Stage; auto result = stage.Ref().GetTypeAnn(); @@ -402,7 +402,7 @@ namespace NYql::NDqs { TProgramBuilder pgmBuilder(typeEnv, *FunctionRegistry); TStringStream errorStream; - auto type = NCommon::BuildType(*exprType, pgmBuilder, errorStream, withTagged); + auto type = NCommon::BuildType(*exprType, pgmBuilder, errorStream, withTagged); return SerializeNode(type, typeEnv); } return {}; @@ -411,12 +411,12 @@ namespace NYql::NDqs { bool TDqsExecutionPlanner::BuildReadStage(const TDqSettings::TPtr& settings, const TDqPhyStage& stage, bool dqSource, bool canFallback) { auto& stageInfo = TasksGraph.GetStageInfo(stage); - for (ui32 i = 0; i < stageInfo.InputsCount; i++) { + for (ui32 i = 0; i < stageInfo.InputsCount; i++) { const auto& input = stage.Inputs().Item(i); YQL_ENSURE(input.Maybe<TDqCnBroadcast>() || (dqSource && input.Maybe<TDqSource>())); - } - - const TExprNode* read = nullptr; + } + + const TExprNode* read = nullptr; ui32 dqSourceInputIndex = std::numeric_limits<ui32>::max(); if (dqSource) { for (ui32 i = 0; i < stageInfo.InputsCount; ++i) { @@ -439,29 +439,29 @@ namespace NYql::NDqs { return true; } return false; - })) { + })) { read = wrap->Child(TDqReadWrapBase::idx_Input); - } else { + } else { return false; - } + } } const ui32 dataSourceChildIndex = dqSource ? 0 : 1; - YQL_ENSURE(read->ChildrenSize() > 1); + YQL_ENSURE(read->ChildrenSize() > 1); YQL_ENSURE(read->Child(dataSourceChildIndex)->IsCallable("DataSource")); auto dataSourceName = read->Child(dataSourceChildIndex)->Child(0)->Content(); - auto datasource = TypeContext->DataSourceMap.FindPtr(dataSourceName); - YQL_ENSURE(datasource); + auto datasource = TypeContext->DataSourceMap.FindPtr(dataSourceName); + YQL_ENSURE(datasource); const auto stageSettings = TDqStageSettings::Parse(stage); auto tasksPerStage = settings->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage); if (stageSettings.IsExternalFunction) { tasksPerStage = Min(tasksPerStage, stageSettings.MaxTransformConcurrency()); } const size_t maxPartitions = stageSettings.SinglePartition ? 1ULL : tasksPerStage; - TVector<TString> parts; - if (auto dqIntegration = (*datasource)->GetDqIntegration()) { - TString clusterName; + TVector<TString> parts; + if (auto dqIntegration = (*datasource)->GetDqIntegration()) { + TString clusterName; _MaxDataSizePerJob = Max(_MaxDataSizePerJob, dqIntegration->Partition(*settings, maxPartitions, *read, parts, &clusterName, ExprContext, canFallback)); TMaybe<::google::protobuf::Any> sourceSettings; TString sourceType; @@ -471,10 +471,10 @@ namespace NYql::NDqs { YQL_ENSURE(!sourceSettings->type_url().empty(), "Data source provider \"" << dataSourceName << "\" did't fill dq source settings for its dq source node"); YQL_ENSURE(sourceType, "Data source provider \"" << dataSourceName << "\" did't fill dq source settings type for its dq source node"); } - for (const auto& p : parts) { + for (const auto& p : parts) { auto& task = TasksGraph.AddTask(stageInfo); - task.Meta.TaskParams[dataSourceName] = p; - task.Meta.ClusterNameHint = clusterName; + task.Meta.TaskParams[dataSourceName] = p; + task.Meta.ClusterNameHint = clusterName; if (dqSource) { task.Inputs[dqSourceInputIndex].SourceSettings = sourceSettings; task.Inputs[dqSourceInputIndex].SourceType = sourceType; @@ -484,7 +484,7 @@ namespace NYql::NDqs { transform.FunctionName = stageSettings.TransformName; } } - return !parts.empty(); + return !parts.empty(); } #define BUILD_CONNECTION(TYPE, BUILDER) \ @@ -513,10 +513,10 @@ namespace NYql::NDqs { #undef BUILD_CONNECTION - THashMap<TStageId, std::tuple<TString,ui64>> TDqsExecutionPlanner::BuildAllPrograms() { + THashMap<TStageId, std::tuple<TString,ui64>> TDqsExecutionPlanner::BuildAllPrograms() { using namespace NKikimr::NMiniKQL; - THashMap<TStageId, std::tuple<TString,ui64>> result; + THashMap<TStageId, std::tuple<TString,ui64>> result; TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), FunctionRegistry->SupportsSizedAllocators()); TTypeEnvironment typeEnv(alloc); TVector<NNodes::TExprBase> fakeReads; @@ -528,33 +528,33 @@ namespace NYql::NDqs { auto paramsType = NDq::CollectParameters(stage.Program(), ExprContext); YQL_ENSURE(paramsType->GetItems().empty()); // TODO support parameters - auto settings = NDq::TDqStageSettings::Parse(stage); - ui64 stageId = stage.Ref().UniqueId(); - auto maybeStageId = PublicIds.find(settings.LogicalId); - if (maybeStageId != PublicIds.end()) { - stageId = maybeStageId->second; - } - -/* TODO: - ui64 stageId = stage.Ref().UniqueId(); - if (auto publicId = TypeContext->TranslateOperationId(stageId)) { - stageId = *publicId; - } - - TExprNode::TPtr lambdaInput = ExprContext.DeepCopyLambda(stage.Program().Ref()); - bool hasNonDeterministicFunctions; - auto status = PeepHoleOptimizeNode<true>(lambdaInput, lambdaInput, ExprContext, *TypeContext, nullptr, hasNonDeterministicFunctions); - if (status != IGraphTransformer::TStatus::Ok) { - ExprContext.AddError(TIssue(ExprContext.GetPosition(lambdaInput->Pos()), TString("Peephole optimization failed for Dq stage"))); - ExprContext.IssueManager.GetIssues().PrintTo(Cerr); - Y_VERIFY(false); - } -*/ - result[stageInfo.first] = std::make_tuple( - NDq::BuildProgram( - stage.Program(), *paramsType, compiler, typeEnv, *FunctionRegistry, - ExprContext, fakeReads), - stageId); + auto settings = NDq::TDqStageSettings::Parse(stage); + ui64 stageId = stage.Ref().UniqueId(); + auto maybeStageId = PublicIds.find(settings.LogicalId); + if (maybeStageId != PublicIds.end()) { + stageId = maybeStageId->second; + } + +/* TODO: + ui64 stageId = stage.Ref().UniqueId(); + if (auto publicId = TypeContext->TranslateOperationId(stageId)) { + stageId = *publicId; + } + + TExprNode::TPtr lambdaInput = ExprContext.DeepCopyLambda(stage.Program().Ref()); + bool hasNonDeterministicFunctions; + auto status = PeepHoleOptimizeNode<true>(lambdaInput, lambdaInput, ExprContext, *TypeContext, nullptr, hasNonDeterministicFunctions); + if (status != IGraphTransformer::TStatus::Ok) { + ExprContext.AddError(TIssue(ExprContext.GetPosition(lambdaInput->Pos()), TString("Peephole optimization failed for Dq stage"))); + ExprContext.IssueManager.GetIssues().PrintTo(Cerr); + Y_VERIFY(false); + } +*/ + result[stageInfo.first] = std::make_tuple( + NDq::BuildProgram( + stage.Program(), *paramsType, compiler, typeEnv, *FunctionRegistry, + ExprContext, fakeReads), + stageId); } return result; @@ -617,7 +617,7 @@ namespace NYql::NDqs { break; case TTaskOutputType::HashPartition: { - YQL_ENSURE(output.Channels.size() == output.PartitionsCount); + YQL_ENSURE(output.Channels.size() == output.PartitionsCount); auto& hashPartitionDesc = *outputDesc.MutableHashPartition(); for (auto& column : output.KeyColumns) { hashPartitionDesc.AddKeyColumns(column); @@ -653,157 +653,157 @@ namespace NYql::NDqs { FillChannelDesc(channelDesc, TasksGraph.GetChannel(channel)); } } - - - TDqsSingleExecutionPlanner::TDqsSingleExecutionPlanner( - const TString& program, - NActors::TActorId executerID, - NActors::TActorId resultID, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const TTypeAnnotationNode* typeAnn) - : Program(program) - , ExecuterID(executerID) - , ResultID(resultID) - , FunctionRegistry(functionRegistry) - , TypeAnn(typeAnn) - { } - - TVector<TDqTask>& TDqsSingleExecutionPlanner::GetTasks() - { - if (Tasks.empty()) { - Tasks = GetTasks({NActors::TActorId(1, 0, 0, 0)}); - } - return Tasks; - } - - TVector<TDqTask> TDqsSingleExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) - { - YQL_ENSURE(workers.size() == 1); - - auto& worker = workers[0]; - - TDqTask task; - Yql::DqsProto::TTaskMeta taskMeta; - task.SetId(1); - taskMeta.SetStageId(1); - task.SetStageId(1); - task.MutableMeta()->PackFrom(taskMeta); - + + + TDqsSingleExecutionPlanner::TDqsSingleExecutionPlanner( + const TString& program, + NActors::TActorId executerID, + NActors::TActorId resultID, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + const TTypeAnnotationNode* typeAnn) + : Program(program) + , ExecuterID(executerID) + , ResultID(resultID) + , FunctionRegistry(functionRegistry) + , TypeAnn(typeAnn) + { } + + TVector<TDqTask>& TDqsSingleExecutionPlanner::GetTasks() + { + if (Tasks.empty()) { + Tasks = GetTasks({NActors::TActorId(1, 0, 0, 0)}); + } + return Tasks; + } + + TVector<TDqTask> TDqsSingleExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) + { + YQL_ENSURE(workers.size() == 1); + + auto& worker = workers[0]; + + TDqTask task; + Yql::DqsProto::TTaskMeta taskMeta; + task.SetId(1); + taskMeta.SetStageId(1); + task.SetStageId(1); + task.MutableMeta()->PackFrom(taskMeta); + NActors::ActorIdToProto(ExecuterID, task.MutableExecuter()->MutableActorId()); - auto& program = *task.MutableProgram(); - program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0); - program.SetRaw(Program); - - auto outputDesc = task.AddOutputs(); - outputDesc->MutableMap(); - - auto channelDesc = outputDesc->AddChannels(); - channelDesc->SetId(1); - channelDesc->SetSrcTaskId(2); - channelDesc->SetDstTaskId(1); - + auto& program = *task.MutableProgram(); + program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0); + program.SetRaw(Program); + + auto outputDesc = task.AddOutputs(); + outputDesc->MutableMap(); + + auto channelDesc = outputDesc->AddChannels(); + channelDesc->SetId(1); + channelDesc->SetSrcTaskId(2); + channelDesc->SetDstTaskId(1); + NActors::ActorIdToProto(ExecuterID, channelDesc->MutableSrcEndpoint()->MutableActorId()); NActors::ActorIdToProto(ResultID, channelDesc->MutableDstEndpoint()->MutableActorId()); - - SourceID = worker; - - return {task}; - } - + + SourceID = worker; + + return {task}; + } + NActors::TActorId TDqsSingleExecutionPlanner::GetSourceID() const - { + { if (SourceID) { return *SourceID; } else { return {}; } - } - - TString TDqsSingleExecutionPlanner::GetResultType(bool withTagged) const - { - if (withTagged && TypeAnn) { - auto item = TypeAnn; - YQL_ENSURE(item->GetKind() == ETypeAnnotationKind::List); - auto exprType = item->Cast<TListExprType>()->GetItemType(); - - TScopedAlloc alloc; - TTypeEnvironment typeEnv(alloc); - - TProgramBuilder pgmBuilder(typeEnv, *FunctionRegistry); - TStringStream errorStream; - auto type = NCommon::BuildType(*exprType, pgmBuilder, errorStream, withTagged); - return SerializeNode(type, typeEnv); - } else { - return GetSerializedResultType(Program); - } - } - - TGraphExecutionPlanner::TGraphExecutionPlanner( - const TVector<TDqTask>& tasks, - ui64 sourceId, - const TString& resultType, + } + + TString TDqsSingleExecutionPlanner::GetResultType(bool withTagged) const + { + if (withTagged && TypeAnn) { + auto item = TypeAnn; + YQL_ENSURE(item->GetKind() == ETypeAnnotationKind::List); + auto exprType = item->Cast<TListExprType>()->GetItemType(); + + TScopedAlloc alloc; + TTypeEnvironment typeEnv(alloc); + + TProgramBuilder pgmBuilder(typeEnv, *FunctionRegistry); + TStringStream errorStream; + auto type = NCommon::BuildType(*exprType, pgmBuilder, errorStream, withTagged); + return SerializeNode(type, typeEnv); + } else { + return GetSerializedResultType(Program); + } + } + + TGraphExecutionPlanner::TGraphExecutionPlanner( + const TVector<TDqTask>& tasks, + ui64 sourceId, + const TString& resultType, NActors::TActorId executerID, NActors::TActorId resultID) - : Tasks(tasks) - , SourceId(sourceId) - , ResultType(resultType) - , ExecuterID(executerID) - , ResultID(resultID) - { - } - - TVector<TDqTask> TGraphExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) - { + : Tasks(tasks) + , SourceId(sourceId) + , ResultType(resultType) + , ExecuterID(executerID) + , ResultID(resultID) + { + } + + TVector<TDqTask> TGraphExecutionPlanner::GetTasks(const TVector<NActors::TActorId>& workers) + { if (ResultType) { YQL_ENSURE(SourceId < workers.size()); SourceID = workers[SourceId]; } - - auto setActorId = [&](NYql::NDqProto::TEndpoint* endpoint) { - if (endpoint->GetEndpointTypeCase() == NYql::NDqProto::TEndpoint::kActorId) { + + auto setActorId = [&](NYql::NDqProto::TEndpoint* endpoint) { + if (endpoint->GetEndpointTypeCase() == NYql::NDqProto::TEndpoint::kActorId) { NActors::TActorId fakeId = NActors::ActorIdFromProto(endpoint->GetActorId()); - if (fakeId.NodeId() > 0) { + if (fakeId.NodeId() > 0) { NActors::TActorId realId = fakeId.LocalId() == 0 - ? workers[fakeId.NodeId()-1] - : ResultID; + ? workers[fakeId.NodeId()-1] + : ResultID; NActors::ActorIdToProto(realId, endpoint->MutableActorId()); - } - } - }; - - for (auto& taskDesc : Tasks) { + } + } + }; + + for (auto& taskDesc : Tasks) { NActors::ActorIdToProto(ExecuterID, taskDesc.MutableExecuter()->MutableActorId()); - - for (auto& inputDesc : *taskDesc.MutableInputs()) { - for (auto& channelDesc : *inputDesc.MutableChannels()) { - setActorId(channelDesc.MutableSrcEndpoint()); - setActorId(channelDesc.MutableDstEndpoint()); - } - } - - for (auto& outputDesc : *taskDesc.MutableOutputs()) { - for (auto& channelDesc : *outputDesc.MutableChannels()) { - setActorId(channelDesc.MutableSrcEndpoint()); - setActorId(channelDesc.MutableDstEndpoint()); - } - } - } - - return Tasks; - } - + + for (auto& inputDesc : *taskDesc.MutableInputs()) { + for (auto& channelDesc : *inputDesc.MutableChannels()) { + setActorId(channelDesc.MutableSrcEndpoint()); + setActorId(channelDesc.MutableDstEndpoint()); + } + } + + for (auto& outputDesc : *taskDesc.MutableOutputs()) { + for (auto& channelDesc : *outputDesc.MutableChannels()) { + setActorId(channelDesc.MutableSrcEndpoint()); + setActorId(channelDesc.MutableDstEndpoint()); + } + } + } + + return Tasks; + } + NActors::TActorId TGraphExecutionPlanner::GetSourceID() const - { + { if (SourceID) { return *SourceID; } else { return {}; } - } - - TString TGraphExecutionPlanner::GetResultType(bool) const - { - return ResultType; - } - -} // namespace NYql::NDqs + } + + TString TGraphExecutionPlanner::GetResultType(bool) const + { + return ResultType; + } + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.h b/ydb/library/yql/providers/dq/planner/execution_planner.h index 803f2083b1..280600c8f7 100644 --- a/ydb/library/yql/providers/dq/planner/execution_planner.h +++ b/ydb/library/yql/providers/dq/planner/execution_planner.h @@ -14,45 +14,45 @@ #include <tuple> namespace NYql::NDqs { - class IDqsExecutionPlanner { + class IDqsExecutionPlanner { public: - virtual ~IDqsExecutionPlanner() = default; - virtual TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) = 0; - virtual TVector<NDqProto::TDqTask>& GetTasks() = 0; + virtual ~IDqsExecutionPlanner() = default; + virtual TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) = 0; + virtual TVector<NDqProto::TDqTask>& GetTasks() = 0; virtual NActors::TActorId GetSourceID() const = 0; - virtual TString GetResultType(bool withTagged = false) const = 0; - }; - - class TDqsExecutionPlanner: public IDqsExecutionPlanner { - public: - explicit TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext, + virtual TString GetResultType(bool withTagged = false) const = 0; + }; + + class TDqsExecutionPlanner: public IDqsExecutionPlanner { + public: + explicit TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext, NYql::TExprContext& exprContext, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NYql::TExprNode::TPtr dqExprRoot, NActors::TActorId executerID = NActors::TActorId(), NActors::TActorId resultID = NActors::TActorId(1, 0, 1, 0)); - void Clear(); - bool CanFallback(); - ui64 MaxDataSizePerJob() { - return _MaxDataSizePerJob; - } - ui64 StagesCount(); + void Clear(); + bool CanFallback(); + ui64 MaxDataSizePerJob() { + return _MaxDataSizePerJob; + } + ui64 StagesCount(); ui32 PlanExecution(const TDqSettings::TPtr& settings, bool canFallback = false); - TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; - TVector<NDqProto::TDqTask>& GetTasks() override; + TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; + TVector<NDqProto::TDqTask>& GetTasks() override; NActors::TActorId GetSourceID() const override; - TString GetResultType(bool withTagged = false) const override; + TString GetResultType(bool withTagged = false) const override; + + void SetPublicIds(const THashMap<ui64, ui32>& publicIds) { + PublicIds = publicIds; + } - void SetPublicIds(const THashMap<ui64, ui32>& publicIds) { - PublicIds = publicIds; - } - private: bool BuildReadStage(const TDqSettings::TPtr& settings, const NNodes::TDqPhyStage& stage, bool dqSource, bool canFallback); void BuildConnections(const NNodes::TDqPhyStage& stage); - THashMap<NDq::TStageId, std::tuple<TString,ui64>> BuildAllPrograms(); + THashMap<NDq::TStageId, std::tuple<TString,ui64>> BuildAllPrograms(); void FillChannelDesc(NDqProto::TChannel& channelDesc, const NDq::TChannel& channel); void FillInputDesc(NDqProto::TTaskInput& inputDesc, const TTaskInput& input); void FillOutputDesc(NDqProto::TTaskOutput& outputDesc, const TTaskOutput& output); @@ -64,72 +64,72 @@ namespace NYql::NDqs { private: TIntrusivePtr<TTypeAnnotationContext> TypeContext; NYql::TExprContext& ExprContext; - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; NYql::TExprNode::TPtr DqExprRoot; TVector<const TTypeAnnotationNode*> InputType; NActors::TActorId ExecuterID; NActors::TActorId ResultID; TMaybe<NActors::TActorId> SourceID = {}; ui64 SourceTaskID = 0; - ui64 _MaxDataSizePerJob = 0; + ui64 _MaxDataSizePerJob = 0; TDqsTasksGraph TasksGraph; - TVector<NDqProto::TDqTask> Tasks; - - THashMap<ui64, ui32> PublicIds; + TVector<NDqProto::TDqTask> Tasks; + + THashMap<ui64, ui32> PublicIds; }; - - // Execution planner for TRuntimeNode - class TDqsSingleExecutionPlanner: public IDqsExecutionPlanner { - public: - TDqsSingleExecutionPlanner( - const TString& program, - NActors::TActorId executerID, - NActors::TActorId resultID, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - const TTypeAnnotationNode* typeAnn); - - TVector<NDqProto::TDqTask>& GetTasks() override; - TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; + + // Execution planner for TRuntimeNode + class TDqsSingleExecutionPlanner: public IDqsExecutionPlanner { + public: + TDqsSingleExecutionPlanner( + const TString& program, + NActors::TActorId executerID, + NActors::TActorId resultID, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + const TTypeAnnotationNode* typeAnn); + + TVector<NDqProto::TDqTask>& GetTasks() override; + TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; NActors::TActorId GetSourceID() const override; - TString GetResultType(bool withTagged = false) const override; - - private: - TString Program; + TString GetResultType(bool withTagged = false) const override; + + private: + TString Program; NActors::TActorId ExecuterID; NActors::TActorId ResultID; - + TMaybe<NActors::TActorId> SourceID = {}; - TVector<NDqProto::TDqTask> Tasks; - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; - const TTypeAnnotationNode* TypeAnn; - }; - - // Execution planner for Graph - class TGraphExecutionPlanner: public IDqsExecutionPlanner { - public: - TGraphExecutionPlanner( - const TVector<NDqProto::TDqTask>& tasks, - ui64 sourceId, - const TString& resultType, + TVector<NDqProto::TDqTask> Tasks; + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + const TTypeAnnotationNode* TypeAnn; + }; + + // Execution planner for Graph + class TGraphExecutionPlanner: public IDqsExecutionPlanner { + public: + TGraphExecutionPlanner( + const TVector<NDqProto::TDqTask>& tasks, + ui64 sourceId, + const TString& resultType, NActors::TActorId executerID, NActors::TActorId resultID); - - TVector<NDqProto::TDqTask>& GetTasks() override { - ythrow yexception() << "unimplemented"; - } - TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; + + TVector<NDqProto::TDqTask>& GetTasks() override { + ythrow yexception() << "unimplemented"; + } + TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override; NActors::TActorId GetSourceID() const override; - TString GetResultType(bool withTagged = false) const override; - - private: - TVector<NDqProto::TDqTask> Tasks; + TString GetResultType(bool withTagged = false) const override; + + private: + TVector<NDqProto::TDqTask> Tasks; ui64 SourceId = 0; - TString ResultType; - + TString ResultType; + NActors::TActorId ExecuterID; NActors::TActorId ResultID; - + TMaybe<NActors::TActorId> SourceID = {}; - }; + }; } diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 77d76ad636..8759f44221 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -1,6 +1,6 @@ #include <ydb/library/yql/providers/dq/provider/yql_dq_datasource.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_state.h> - + #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> @@ -8,7 +8,7 @@ #include <ydb/library/yql/providers/common/transform/yql_lazy_init.h> #include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h> #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> - + #include <ydb/library/yql/providers/dq/opt/dqs_opt.h> #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/dq/actors/proto_builder.h> @@ -18,144 +18,144 @@ #include <ydb/library/yql/providers/dq/planner/execution_planner.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_control.h> - + #include <ydb/library/yql/dq/type_ann/dq_type_ann.h> #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> #include <ydb/library/yql/dq/tasks/dq_task_program.h> #include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h> #include <ydb/library/yql/dq/opt/dq_opt_build.h> #include <ydb/library/yql/dq/opt/dq_opt.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/services/yql_transform_pipeline.h> #include <ydb/library/yql/core/services/yql_out_transformers.h> - + #include <ydb/library/yql/minikql/mkql_alloc.h> #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> #include <ydb/library/yql/minikql/aligned_page_pool.h> - + #include <ydb/library/yql/core/type_ann/type_ann_expr.h> #include <ydb/library/yql/core/yql_type_annotation.h> #include <ydb/library/yql/core/yql_graph_transformer.h> #include <ydb/library/yql/core/yql_opt_utils.h> #include <ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h> - -#include <library/cpp/yson/node/node_io.h> -#include <library/cpp/svnversion/svnversion.h> -#include <library/cpp/digest/md5/md5.h> - -#include <util/system/env.h> + +#include <library/cpp/yson/node/node_io.h> +#include <library/cpp/svnversion/svnversion.h> +#include <library/cpp/digest/md5/md5.h> + +#include <util/system/env.h> #include <util/generic/size_literals.h> #include <util/stream/file.h> #include <util/string/builder.h> -namespace NYql { - -using namespace NCommon; -using namespace NKikimr::NMiniKQL; -using namespace NNodes; - +namespace NYql { + +using namespace NCommon; +using namespace NKikimr::NMiniKQL; +using namespace NNodes; + namespace { -using TUploadList = IDqGateway::TUploadList; - -// TODO: move this to separate file -class TLocalExecutor: public TCounters -{ -public: - TLocalExecutor(const TDqStatePtr& state) - : State(state) - { } - +using TUploadList = IDqGateway::TUploadList; + +// TODO: move this to separate file +class TLocalExecutor: public TCounters +{ +public: + TLocalExecutor(const TDqStatePtr& state) + : State(state) + { } + NThreading::TFuture<IDqGateway::TResult> Execute(TPosition pos, const TString& lambda, const TVector<TString>& columns, - const THashMap<TString, TString>& secureParams, const IDataProvider::TFillSettings& fillSettings) - { - try { - return ExecuteUnsafe(lambda, columns, secureParams, fillSettings); + const THashMap<TString, TString>& secureParams, const IDataProvider::TFillSettings& fillSettings) + { + try { + return ExecuteUnsafe(lambda, columns, secureParams, fillSettings); } catch (const NKikimr::TMemoryLimitExceededException& e) { auto res = ResultFromError<IDqGateway::TResult>(TStringBuilder() << "DQ computation exceeds the memory limit " << State->Settings->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit", pos); return NThreading::MakeFuture(res); } catch (const std::exception& e) { return NThreading::MakeFuture(ResultFromException<IDqGateway::TResult>(e, pos)); - } catch (...) { + } catch (...) { auto res = ResultFromError<IDqGateway::TResult>(CurrentExceptionMessage(), pos); res.SetStatus(TIssuesIds::UNEXPECTED); return NThreading::MakeFuture(res); - } - } - + } + } + NThreading::TFuture<IDqGateway::TResult> ExecuteUnsafe(const TString& lambda, const TVector<TString>& columns, - const THashMap<TString, TString>& secureParams, const IDataProvider::TFillSettings& fillSettings) - { - auto t = TInstant::Now(); - IDqGateway::TResult result; - NDqProto::TDqTask task; - task.SetId(0); + const THashMap<TString, TString>& secureParams, const IDataProvider::TFillSettings& fillSettings) + { + auto t = TInstant::Now(); + IDqGateway::TResult result; + NDqProto::TDqTask task; + task.SetId(0); task.SetStageId(0); - - auto& program = *task.MutableProgram(); - program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0); - program.SetRaw(lambda); - - auto outputDesc = task.AddOutputs(); - outputDesc->MutableMap(); - - auto channelDesc = outputDesc->AddChannels(); - channelDesc->SetId(0); - channelDesc->SetSrcTaskId(1); - channelDesc->SetDstTaskId(0); - - // TODO: remove this - auto deterministicMode = !!GetEnv("YQL_DETERMINISTIC_MODE"); - auto timeProvider = deterministicMode - ? CreateDeterministicTimeProvider(10000000) - : CreateDefaultTimeProvider(); - auto randomProvider = deterministicMode - ? CreateDeterministicRandomProvider(1) - : State->RandomProvider; - - NDq::TDqTaskRunnerContext executionContext; - executionContext.FuncRegistry = State->FunctionRegistry; - + + auto& program = *task.MutableProgram(); + program.SetRuntimeVersion(NYql::NDqProto::ERuntimeVersion::RUNTIME_VERSION_YQL_1_0); + program.SetRaw(lambda); + + auto outputDesc = task.AddOutputs(); + outputDesc->MutableMap(); + + auto channelDesc = outputDesc->AddChannels(); + channelDesc->SetId(0); + channelDesc->SetSrcTaskId(1); + channelDesc->SetDstTaskId(0); + + // TODO: remove this + auto deterministicMode = !!GetEnv("YQL_DETERMINISTIC_MODE"); + auto timeProvider = deterministicMode + ? CreateDeterministicTimeProvider(10000000) + : CreateDefaultTimeProvider(); + auto randomProvider = deterministicMode + ? CreateDeterministicRandomProvider(1) + : State->RandomProvider; + + NDq::TDqTaskRunnerContext executionContext; + executionContext.FuncRegistry = State->FunctionRegistry; + executionContext.ComputationFactory = State->ComputationFactory; - executionContext.RandomProvider = randomProvider.Get(); - executionContext.TimeProvider = timeProvider.Get(); - - NDq::TDqTaskRunnerMemoryLimits limits; + executionContext.RandomProvider = randomProvider.Get(); + executionContext.TimeProvider = timeProvider.Get(); + + NDq::TDqTaskRunnerMemoryLimits limits; limits.ChannelBufferSize = 10_MB; limits.OutputChunkMaxSize = 2_MB; - + NDq::TDqTaskRunnerSettings settings; settings.OptLLVM = "OFF"; // Don't use LLVM for local execution settings.SecureParams = secureParams; - settings.CollectBasicStats = true; - settings.CollectProfileStats = true; + settings.CollectBasicStats = true; + settings.CollectProfileStats = true; settings.AllowGeneratorsInUnboxedValues = true; auto runner = NDq::MakeDqTaskRunner(executionContext, settings, {}); - + { auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0)); runner->Prepare(task, limits); } - TVector<NDqProto::TData> rows; - { + TVector<NDqProto::TData> rows; + { auto guard = runner->BindAllocator(State->Settings->MemoryLimit.Get().GetOrElse(0)); - YQL_LOG(DEBUG) << " NDq::ERunStatus " << runner->Run(); - - NDq::ERunStatus status; - while ((status = runner->Run()) == NDq::ERunStatus::PendingOutput || status == NDq::ERunStatus::Finished) { - NDqProto::TData data; + YQL_LOG(DEBUG) << " NDq::ERunStatus " << runner->Run(); + + NDq::ERunStatus status; + while ((status = runner->Run()) == NDq::ERunStatus::PendingOutput || status == NDq::ERunStatus::Finished) { + NDqProto::TData data; if (runner->GetOutputChannel(0)->PopAll(data) && !fillSettings.Discard) { - rows.push_back(data); - } - if (status == NDq::ERunStatus::Finished) { - break; - } + rows.push_back(data); + } + if (status == NDq::ERunStatus::Finished) { + break; + } if (!fillSettings.Discard) { if (fillSettings.AllResultsBytesLimit && runner->GetOutputChannel(0)->GetStats()->Bytes >= *fillSettings.AllResultsBytesLimit) { result.Truncated = true; @@ -165,47 +165,47 @@ public: result.Truncated = true; break; } - } - } - - YQL_ENSURE(status == NDq::ERunStatus::Finished || status == NDq::ERunStatus::PendingOutput); - } - - auto serializedResultType = GetSerializedResultType(lambda); + } + } + + YQL_ENSURE(status == NDq::ERunStatus::Finished || status == NDq::ERunStatus::PendingOutput); + } + + auto serializedResultType = GetSerializedResultType(lambda); NYql::NDqs::TProtoBuilder protoBuilder(serializedResultType, columns); - + result.Data = protoBuilder.BuildYson(rows); - - AddCounter("LocalRun", TInstant::Now() - t); - - - FlushStatisticsToState(); - - // TODO: move this to separate thread-pool - auto promise = NThreading::NewPromise<IDqGateway::TResult>(); - auto future = promise.GetFuture(); - - result.SetSuccess(); - - promise.SetValue(result); - - return future; - } - -private: - void FlushStatisticsToState() { - TOperationStatistics statistics; - FlushCounters(statistics); - - TGuard<TMutex> lock(State->Mutex); - if (!statistics.Entries.empty()) { - State->Statistics[State->MetricId++] = statistics; - } - } - - TDqStatePtr State; -}; - + + AddCounter("LocalRun", TInstant::Now() - t); + + + FlushStatisticsToState(); + + // TODO: move this to separate thread-pool + auto promise = NThreading::NewPromise<IDqGateway::TResult>(); + auto future = promise.GetFuture(); + + result.SetSuccess(); + + promise.SetValue(result); + + return future; + } + +private: + void FlushStatisticsToState() { + TOperationStatistics statistics; + FlushCounters(statistics); + + TGuard<TMutex> lock(State->Mutex); + if (!statistics.Entries.empty()) { + State->Statistics[State->MetricId++] = statistics; + } + } + + TDqStatePtr State; +}; + struct TDqsPipelineConfigurator : public IPipelineConfigurator { private: void AfterCreate(TTransformationPipeline*) const final {} @@ -218,123 +218,123 @@ private: void AfterOptimize(TTransformationPipeline*) const final {} }; -class TInMemoryExecTransformer: public TExecTransformerBase, TCounters -{ -public: - TInMemoryExecTransformer(const TDqStatePtr& state) - : State(state) - , DqTypeAnnotationTransformer( - CreateTypeAnnotationTransformer(NDq::CreateDqTypeAnnotationTransformer(*State->TypeCtx), *State->TypeCtx)) - { +class TInMemoryExecTransformer: public TExecTransformerBase, TCounters +{ +public: + TInMemoryExecTransformer(const TDqStatePtr& state) + : State(state) + , DqTypeAnnotationTransformer( + CreateTypeAnnotationTransformer(NDq::CreateDqTypeAnnotationTransformer(*State->TypeCtx), *State->TypeCtx)) + { AddHandler({TStringBuf("Result")}, RequireNone(), Hndl(&TInMemoryExecTransformer::HandleResult)); AddHandler({TStringBuf("Pull")}, RequireNone(), Hndl(&TInMemoryExecTransformer::HandlePull)); - AddHandler({TDqCnResult::CallableName()}, RequireNone(), Pass()); + AddHandler({TDqCnResult::CallableName()}, RequireNone(), Pass()); AddHandler({TDqQuery::CallableName()}, RequireFirst(), Pass()); - } - -private: - void GetResultType(TString* type, TVector<TString>* columns, const TExprNode& resOrPull, const TExprNode& resOrPullInput) const - { - *columns = NCommon::GetResOrPullColumnHints(resOrPull); - if (columns->empty()) { - *columns = NCommon::GetStructFields(resOrPullInput.GetTypeAnn()); - } - - if (NCommon::HasResOrPullOption(resOrPull, "type")) { - TStringStream typeYson; + } + +private: + void GetResultType(TString* type, TVector<TString>* columns, const TExprNode& resOrPull, const TExprNode& resOrPullInput) const + { + *columns = NCommon::GetResOrPullColumnHints(resOrPull); + if (columns->empty()) { + *columns = NCommon::GetStructFields(resOrPullInput.GetTypeAnn()); + } + + if (NCommon::HasResOrPullOption(resOrPull, "type")) { + TStringStream typeYson; NYson::TYsonWriter typeWriter(&typeYson); - NCommon::WriteResOrPullType(typeWriter, resOrPullInput.GetTypeAnn(), *columns); - *type = typeYson.Str(); - } - } - + NCommon::WriteResOrPullType(typeWriter, resOrPullInput.GetTypeAnn(), *columns); + *type = typeYson.Str(); + } + } + TExprNode::TPtr GetLambdaBody(int& level, TExprNode::TPtr&& node, TExprContext& ctx) const { const auto kind = node->GetTypeAnn()->GetKind(); const bool data = kind != ETypeAnnotationKind::Flow && kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Stream && kind != ETypeAnnotationKind::Optional; level = data ? 1 : 0; return ctx.WrapByCallableIf(kind != ETypeAnnotationKind::Stream, "ToStream", ctx.WrapByCallableIf(data, "Just", std::move(node))); - } - - std::tuple<TString, TString> GetPathAndObjectId(const TString& path, const TString& objectId, const TString& md5) const { - if (path.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix) - || !State->Settings->EnableStrip.Get() || !State->Settings->EnableStrip.Get().GetOrElse(false)) - { - ModulesMapping.emplace(objectId, path); - return std::make_tuple(path, objectId); - } - - TFileLinkPtr& fileLink = FileLinks[objectId]; - if (!fileLink) { - fileLink = State->FileStorage->PutFileStripped(path, md5); - } - - ModulesMapping.emplace(objectId + DqStrippedSuffied, path); - - return std::make_tuple(fileLink->GetPath(), objectId + DqStrippedSuffied); - } - - std::tuple<TString, TString> GetPathAndObjectId(const TFilePathWithMd5& pathWithMd5) const { - if (pathWithMd5.Md5.empty()) { - YQL_LOG(WARN) << "Empty md5 for " << pathWithMd5.Path; - } - return GetPathAndObjectId(pathWithMd5.Path, - pathWithMd5.Md5.empty() - ? MD5::File(pathWithMd5.Path) /* used for local run only */ - : pathWithMd5.Md5, - pathWithMd5.Md5); - } - - bool BuildUploadList( - TUploadList* uploadList, - bool localRun, - TString* lambda, - TTypeEnvironment& typeEnv, - TUserDataTable& files) const - { - auto root = DeserializeRuntimeNode(*lambda, typeEnv); - TExploringNodeVisitor explorer; - explorer.Walk(root.GetNode(), typeEnv); - auto ret = BuildUploadList(uploadList, localRun, explorer, typeEnv, files); - *lambda = SerializeRuntimeNode(root, typeEnv); - return ret; - } - - bool BuildUploadList( - TUploadList* uploadList, - bool localRun, - TExploringNodeVisitor& explorer, - TTypeEnvironment& typeEnv, - TUserDataTable& files) const - { - if (State->VanillaJobPath.empty()) { - auto f = IDqGateway::TFileResource(); - f.SetName("dq_vanilla_job.lite"); - f.SetObjectId(GetProgramCommitId()); - f.SetObjectType(IDqGateway::TFileResource::EEXE_FILE); - uploadList->emplace(f); - } else { - auto f = IDqGateway::TFileResource(); - f.SetName("dq_vanilla_job.lite"); - TString path = State->VanillaJobPath; - TString objectId = GetProgramCommitId(); - std::tie(path, objectId) = GetPathAndObjectId(path, objectId, State->VanillaJobMd5); - f.SetObjectId(objectId); - f.SetLocalPath(path); - f.SetObjectType(IDqGateway::TFileResource::EEXE_FILE); - f.SetSize(TFile(path, OpenExisting | RdOnly).GetLength()); - uploadList->emplace(f); - } - - for (TNode* node : explorer.GetNodes()) { - node->Freeze(typeEnv); - - if (node->GetType()->IsCallable()) { - auto& callable = static_cast<NKikimr::NMiniKQL::TCallable&>(*node); - if (!callable.HasResult()) { - const auto& callableType = callable.GetType(); - const auto& name = callableType->GetNameStr(); + } + + std::tuple<TString, TString> GetPathAndObjectId(const TString& path, const TString& objectId, const TString& md5) const { + if (path.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix) + || !State->Settings->EnableStrip.Get() || !State->Settings->EnableStrip.Get().GetOrElse(false)) + { + ModulesMapping.emplace(objectId, path); + return std::make_tuple(path, objectId); + } + + TFileLinkPtr& fileLink = FileLinks[objectId]; + if (!fileLink) { + fileLink = State->FileStorage->PutFileStripped(path, md5); + } + + ModulesMapping.emplace(objectId + DqStrippedSuffied, path); + + return std::make_tuple(fileLink->GetPath(), objectId + DqStrippedSuffied); + } + + std::tuple<TString, TString> GetPathAndObjectId(const TFilePathWithMd5& pathWithMd5) const { + if (pathWithMd5.Md5.empty()) { + YQL_LOG(WARN) << "Empty md5 for " << pathWithMd5.Path; + } + return GetPathAndObjectId(pathWithMd5.Path, + pathWithMd5.Md5.empty() + ? MD5::File(pathWithMd5.Path) /* used for local run only */ + : pathWithMd5.Md5, + pathWithMd5.Md5); + } + + bool BuildUploadList( + TUploadList* uploadList, + bool localRun, + TString* lambda, + TTypeEnvironment& typeEnv, + TUserDataTable& files) const + { + auto root = DeserializeRuntimeNode(*lambda, typeEnv); + TExploringNodeVisitor explorer; + explorer.Walk(root.GetNode(), typeEnv); + auto ret = BuildUploadList(uploadList, localRun, explorer, typeEnv, files); + *lambda = SerializeRuntimeNode(root, typeEnv); + return ret; + } + + bool BuildUploadList( + TUploadList* uploadList, + bool localRun, + TExploringNodeVisitor& explorer, + TTypeEnvironment& typeEnv, + TUserDataTable& files) const + { + if (State->VanillaJobPath.empty()) { + auto f = IDqGateway::TFileResource(); + f.SetName("dq_vanilla_job.lite"); + f.SetObjectId(GetProgramCommitId()); + f.SetObjectType(IDqGateway::TFileResource::EEXE_FILE); + uploadList->emplace(f); + } else { + auto f = IDqGateway::TFileResource(); + f.SetName("dq_vanilla_job.lite"); + TString path = State->VanillaJobPath; + TString objectId = GetProgramCommitId(); + std::tie(path, objectId) = GetPathAndObjectId(path, objectId, State->VanillaJobMd5); + f.SetObjectId(objectId); + f.SetLocalPath(path); + f.SetObjectType(IDqGateway::TFileResource::EEXE_FILE); + f.SetSize(TFile(path, OpenExisting | RdOnly).GetLength()); + uploadList->emplace(f); + } + + for (TNode* node : explorer.GetNodes()) { + node->Freeze(typeEnv); + + if (node->GetType()->IsCallable()) { + auto& callable = static_cast<NKikimr::NMiniKQL::TCallable&>(*node); + if (!callable.HasResult()) { + const auto& callableType = callable.GetType(); + const auto& name = callableType->GetNameStr(); if (name == TStringBuf("FolderPath")) - { + { const TString folderName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); auto blocks = TUserDataStorage::FindUserDataFolder(files, folderName); MKQL_ENSURE(blocks, "Folder not found: " << folderName); @@ -358,9 +358,9 @@ private: f.SetLocalPath(filePath); f.SetName(fullFileName); f.SetObjectId(block->FrozenFile->GetMd5()); - f.SetSize(block->FrozenFile->GetSize()); + f.SetSize(block->FrozenFile->GetSize()); f.SetObjectType(IDqGateway::TFileResource::EUSER_FILE); - uploadList->emplace(f); + uploadList->emplace(f); } const TProgramBuilder pgmBuilder(typeEnv, *State->FunctionRegistry); auto result = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>(folderName); @@ -369,111 +369,111 @@ private: callable.SetResult(result, typeEnv); } } else if (name == TStringBuf("FileContent") || name == TStringBuf("FilePath")) { - const TString fileName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); - - auto block = TUserDataStorage::FindUserDataBlock(files, fileName); - MKQL_ENSURE(block, "File not found: " << fileName); - - auto filePath = block->FrozenFile->GetPath().GetPath(); + const TString fileName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); + + auto block = TUserDataStorage::FindUserDataBlock(files, fileName); + MKQL_ENSURE(block, "File not found: " << fileName); + + auto filePath = block->FrozenFile->GetPath().GetPath(); auto fullFileName = localRun ? filePath : fileName; - - const TProgramBuilder pgmBuilder(typeEnv, *State->FunctionRegistry); - TRuntimeNode result; - switch (block->Type) { - case EUserDataType::URL: - case EUserDataType::PATH: { + + const TProgramBuilder pgmBuilder(typeEnv, *State->FunctionRegistry); + TRuntimeNode result; + switch (block->Type) { + case EUserDataType::URL: + case EUserDataType::PATH: { TString content = (name == TStringBuf("FilePath")) - ? fullFileName - : TFileInput(block->FrozenFile->GetPath()).ReadAll(); - result = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>(content); - break; - } - case EUserDataType::RAW_INLINE_DATA: { + ? fullFileName + : TFileInput(block->FrozenFile->GetPath()).ReadAll(); + result = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>(content); + break; + } + case EUserDataType::RAW_INLINE_DATA: { TString content = (name == TStringBuf("FilePath")) - ? fullFileName - : block->Data; - result = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>(content); - break; - } - default: - YQL_ENSURE(false, "Unknown block type " << block->Type); - } - result.Freeze(); - if (result.GetNode() != node) { - callable.SetResult(result, typeEnv); - } + ? fullFileName + : block->Data; + result = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>(content); + break; + } + default: + YQL_ENSURE(false, "Unknown block type " << block->Type); + } + result.Freeze(); + if (result.GetNode() != node) { + callable.SetResult(result, typeEnv); + } if (name == TStringBuf("FilePath")) { - // filePath, fileName, md5 - auto f = IDqGateway::TFileResource(); - f.SetLocalPath(filePath); - f.SetName(fullFileName); - f.SetObjectId(block->FrozenFile->GetMd5()); - f.SetObjectType(IDqGateway::TFileResource::EUSER_FILE); - f.SetSize(block->FrozenFile->GetSize()); - uploadList->emplace(f); - } + // filePath, fileName, md5 + auto f = IDqGateway::TFileResource(); + f.SetLocalPath(filePath); + f.SetName(fullFileName); + f.SetObjectId(block->FrozenFile->GetMd5()); + f.SetObjectType(IDqGateway::TFileResource::EUSER_FILE); + f.SetSize(block->FrozenFile->GetSize()); + uploadList->emplace(f); + } } else if (name == TStringBuf("Udf") || name == TStringBuf("ScriptUdf")) { - const TString udfName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); - const auto moduleName = ModuleName(udfName); - - YQL_LOG(DEBUG) << "Try to resolve " << moduleName; - TMaybe<TFilePathWithMd5> udfPathWithMd5 = State->TypeCtx->UdfResolver->GetSystemModulePath(moduleName); - YQL_ENSURE(udfPathWithMd5.Defined()); - - TString filePath, objectId; - std::tie(filePath, objectId) = GetPathAndObjectId(*udfPathWithMd5); - - YQL_LOG(DEBUG) << "File|Md5 " << filePath << "|" << objectId; - - if (!filePath.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix)) { - auto f = IDqGateway::TFileResource(); - f.SetLocalPath(filePath); - f.SetName(ToString(moduleName)); - f.SetObjectId(objectId); - f.SetObjectType(IDqGateway::TFileResource::EUDF_FILE); - f.SetSize(TFile(filePath, OpenExisting | RdOnly).GetLength()); - uploadList->emplace(f); - } - + const TString udfName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); + const auto moduleName = ModuleName(udfName); + + YQL_LOG(DEBUG) << "Try to resolve " << moduleName; + TMaybe<TFilePathWithMd5> udfPathWithMd5 = State->TypeCtx->UdfResolver->GetSystemModulePath(moduleName); + YQL_ENSURE(udfPathWithMd5.Defined()); + + TString filePath, objectId; + std::tie(filePath, objectId) = GetPathAndObjectId(*udfPathWithMd5); + + YQL_LOG(DEBUG) << "File|Md5 " << filePath << "|" << objectId; + + if (!filePath.StartsWith(NKikimr::NMiniKQL::StaticModulePrefix)) { + auto f = IDqGateway::TFileResource(); + f.SetLocalPath(filePath); + f.SetName(ToString(moduleName)); + f.SetObjectId(objectId); + f.SetObjectType(IDqGateway::TFileResource::EUDF_FILE); + f.SetSize(TFile(filePath, OpenExisting | RdOnly).GetLength()); + uploadList->emplace(f); + } + if (moduleName == TStringBuf("Geo")) { - TString fileName = "/home/geodata6.bin"; - auto block = TUserDataStorage::FindUserDataBlock(files, fileName); - MKQL_ENSURE(block, "File not found: " << fileName); - auto f = IDqGateway::TFileResource(); - f.SetLocalPath(block->FrozenFile->GetPath().GetPath()); - f.SetName(fileName); - f.SetObjectId(block->FrozenFile->GetMd5()); - f.SetObjectType(IDqGateway::TFileResource::EUSER_FILE); - f.SetSize(block->FrozenFile->GetSize()); - uploadList->emplace(f); - } - } - } - } - } - - i64 sizeSum = 0; - for (const auto& f : *uploadList) { - sizeSum += f.GetSize(); - } - - i64 dataLimit = static_cast<i64>(4_GB); + TString fileName = "/home/geodata6.bin"; + auto block = TUserDataStorage::FindUserDataBlock(files, fileName); + MKQL_ENSURE(block, "File not found: " << fileName); + auto f = IDqGateway::TFileResource(); + f.SetLocalPath(block->FrozenFile->GetPath().GetPath()); + f.SetName(fileName); + f.SetObjectId(block->FrozenFile->GetMd5()); + f.SetObjectType(IDqGateway::TFileResource::EUSER_FILE); + f.SetSize(block->FrozenFile->GetSize()); + uploadList->emplace(f); + } + } + } + } + } + + i64 sizeSum = 0; + for (const auto& f : *uploadList) { + sizeSum += f.GetSize(); + } + + i64 dataLimit = static_cast<i64>(4_GB); bool fallbackFlag = false; - if (sizeSum > dataLimit) { - YQL_LOG(INFO) << "Too much data: " << sizeSum << " > " << dataLimit; - fallbackFlag = true; - } - - return fallbackFlag; - } - - TStatusCallbackPair GetLambda( - TString* lambda, - bool* untrustedUdfFlag, - int* level, - TUploadList* uploadList, - const TResult& result, TExprContext& ctx, bool hasGraphParams) const - { + if (sizeSum > dataLimit) { + YQL_LOG(INFO) << "Too much data: " << sizeSum << " > " << dataLimit; + fallbackFlag = true; + } + + return fallbackFlag; + } + + TStatusCallbackPair GetLambda( + TString* lambda, + bool* untrustedUdfFlag, + int* level, + TUploadList* uploadList, + const TResult& result, TExprContext& ctx, bool hasGraphParams) const + { auto input = Build<TDqPhyStage>(ctx, result.Pos()) .Inputs() .Build() @@ -484,177 +484,177 @@ private: .Settings().Build() .Done().Ptr(); - { - auto block = MeasureBlock("PeepHole"); - - bool hasNonDeterministicFunctions = false; + { + auto block = MeasureBlock("PeepHole"); + + bool hasNonDeterministicFunctions = false; if (const auto status = PeepHoleOptimizeNode<true>(input, input, ctx, *State->TypeCtx, nullptr, hasNonDeterministicFunctions); status.Level != TStatus::Ok) { - return SyncStatus(status); - } - } - - // copy-paste { - TUserDataTable crutches = State->TypeCtx->UserDataStorageCrutches; - TUserDataTable files; - StartCounter("FreezeUsedFiles"); + return SyncStatus(status); + } + } + + // copy-paste { + TUserDataTable crutches = State->TypeCtx->UserDataStorageCrutches; + TUserDataTable files; + StartCounter("FreezeUsedFiles"); if (const auto filesRes = NCommon::FreezeUsedFiles(*input, files, *State->TypeCtx, ctx, [](const TString&){return true;}, crutches); filesRes.first.Level != TStatus::Ok) { - if (filesRes.first.Level != TStatus::Error) { - YQL_LOG(DEBUG) << "Freezing files for " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; - } - return filesRes; - } - FlushCounter("FreezeUsedFiles"); - // copy-paste } - - TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), State->FunctionRegistry->SupportsSizedAllocators()); - TTypeEnvironment typeEnv(alloc); - NCommon::TMkqlCommonCallableCompiler compiler; - - { - auto block = MeasureBlock("BuildProgram"); + if (filesRes.first.Level != TStatus::Error) { + YQL_LOG(DEBUG) << "Freezing files for " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; + } + return filesRes; + } + FlushCounter("FreezeUsedFiles"); + // copy-paste } + + TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), State->FunctionRegistry->SupportsSizedAllocators()); + TTypeEnvironment typeEnv(alloc); + NCommon::TMkqlCommonCallableCompiler compiler; + + { + auto block = MeasureBlock("BuildProgram"); auto programLambda = TDqPhyStage(input).Program(); - - TVector<TExprBase> fakeReads; - auto paramsType = NDq::CollectParameters(programLambda, ctx); - *lambda = NDq::BuildProgram( - programLambda, *paramsType, compiler, typeEnv, *State->FunctionRegistry, - ctx, fakeReads); - } - - auto block = MeasureBlock("RuntimeNodeVisitor"); - - auto root = DeserializeRuntimeNode(*lambda, typeEnv); - - TExploringNodeVisitor explorer; - explorer.Walk(root.GetNode(), typeEnv); - *untrustedUdfFlag = false; - - for (TNode* node : explorer.GetNodes()) { - if (node->GetType()->IsCallable()) { - auto& callable = static_cast<NKikimr::NMiniKQL::TCallable&>(*node); - if (!callable.HasResult()) { - const auto& callableType = callable.GetType(); - const auto& name = callableType->GetNameStr(); - + + TVector<TExprBase> fakeReads; + auto paramsType = NDq::CollectParameters(programLambda, ctx); + *lambda = NDq::BuildProgram( + programLambda, *paramsType, compiler, typeEnv, *State->FunctionRegistry, + ctx, fakeReads); + } + + auto block = MeasureBlock("RuntimeNodeVisitor"); + + auto root = DeserializeRuntimeNode(*lambda, typeEnv); + + TExploringNodeVisitor explorer; + explorer.Walk(root.GetNode(), typeEnv); + *untrustedUdfFlag = false; + + for (TNode* node : explorer.GetNodes()) { + if (node->GetType()->IsCallable()) { + auto& callable = static_cast<NKikimr::NMiniKQL::TCallable&>(*node); + if (!callable.HasResult()) { + const auto& callableType = callable.GetType(); + const auto& name = callableType->GetNameStr(); + if (name == TStringBuf("Udf") || name == TStringBuf("ScriptUdf")) { - const TString udfName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); - const auto moduleName = ModuleName(udfName); - - *untrustedUdfFlag = *untrustedUdfFlag || + const TString udfName(AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().AsStringRef()); + const auto moduleName = ModuleName(udfName); + + *untrustedUdfFlag = *untrustedUdfFlag || callable.GetType()->GetName() == TStringBuf("ScriptUdf") || - !State->FunctionRegistry->IsLoadedUdfModule(moduleName) || + !State->FunctionRegistry->IsLoadedUdfModule(moduleName) || moduleName == TStringBuf("Geo"); - } - } - } - } - - bool localRun = !State->DqGateway || (!*untrustedUdfFlag && !State->TypeCtx->ForceDq && !hasGraphParams); - bool fallbackFlag = BuildUploadList(uploadList, localRun, explorer, typeEnv, files); - - if (fallbackFlag) { - YQL_LOG(DEBUG) << "Fallback: " << NCommon::ExprToPrettyString(ctx, *input); - return Fallback(); - } else { - *lambda = SerializeRuntimeNode(root, typeEnv); - - return SyncStatus(TStatus::Ok); - } - } - + } + } + } + } + + bool localRun = !State->DqGateway || (!*untrustedUdfFlag && !State->TypeCtx->ForceDq && !hasGraphParams); + bool fallbackFlag = BuildUploadList(uploadList, localRun, explorer, typeEnv, files); + + if (fallbackFlag) { + YQL_LOG(DEBUG) << "Fallback: " << NCommon::ExprToPrettyString(ctx, *input); + return Fallback(); + } else { + *lambda = SerializeRuntimeNode(root, typeEnv); + + return SyncStatus(TStatus::Ok); + } + } + static TStatus FallbackCallback(const TDqStatePtr& state, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) - { + { if (state->Metrics) { state->Metrics->IncCounter("dq", "Fallback"); - } + } state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1)); - - YQL_ENSURE(input->ChildrenSize() > 0, "Node: " << NCommon::ExprToPrettyString(ctx, *input)); - TExprNode::TPtr resFill = input->TailPtr(); - output = input; - if (resFill->IsCallable("ResFill!")) { - // TODO: change result provider to remove this if - ui32 inMemoryIndex; - for (inMemoryIndex = 0; inMemoryIndex < resFill->ChildrenSize(); ++inMemoryIndex) { + + YQL_ENSURE(input->ChildrenSize() > 0, "Node: " << NCommon::ExprToPrettyString(ctx, *input)); + TExprNode::TPtr resFill = input->TailPtr(); + output = input; + if (resFill->IsCallable("ResFill!")) { + // TODO: change result provider to remove this if + ui32 inMemoryIndex; + for (inMemoryIndex = 0; inMemoryIndex < resFill->ChildrenSize(); ++inMemoryIndex) { if (resFill->ChildPtr(inMemoryIndex)->IsAtom(DqProviderName)) { - break; - } - } - - YQL_ENSURE(inMemoryIndex != resFill->ChildrenSize(), "Node: " << NCommon::ExprToPrettyString(ctx, *input)); + break; + } + } + + YQL_ENSURE(inMemoryIndex != resFill->ChildrenSize(), "Node: " << NCommon::ExprToPrettyString(ctx, *input)); YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty()); YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName); - + auto newAtom = ctx.NewAtom(input->Pos(), state->TypeCtx->AvailablePureResultDataSources.front()); - resFill->Child(inMemoryIndex)->SetState(TExprNode::EState::ExecutionComplete); - resFill = ctx.ChangeChild(*resFill, inMemoryIndex, std::move(newAtom)); - - input->Child(input->ChildrenSize()-1)->SetState(TExprNode::EState::ExecutionComplete); - output = ctx.ChangeChild(*input, input->ChildrenSize()-1, std::move(resFill)); - return TStatus::Repeat; - } else { + resFill->Child(inMemoryIndex)->SetState(TExprNode::EState::ExecutionComplete); + resFill = ctx.ChangeChild(*resFill, inMemoryIndex, std::move(newAtom)); + + input->Child(input->ChildrenSize()-1)->SetState(TExprNode::EState::ExecutionComplete); + output = ctx.ChangeChild(*input, input->ChildrenSize()-1, std::move(resFill)); + return TStatus::Repeat; + } else { YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty()); YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName); - - TStringStream out; + + TStringStream out; NYson::TYsonWriter writer((IOutputStream*)&out); - writer.OnBeginMap(); - writer.OnKeyedItem("FallbackProvider"); + writer.OnBeginMap(); + writer.OnKeyedItem("FallbackProvider"); writer.OnRaw(state->TypeCtx->AvailablePureResultDataSources.front()); - writer.OnEndMap(); - - output->SetResult(ctx.NewAtom(input->Pos(), out.Str())); - return TStatus::Ok; - } - } - - TStatusCallbackPair Fallback() const { + writer.OnEndMap(); + + output->SetResult(ctx.NewAtom(input->Pos(), out.Str())); + return TStatus::Ok; + } + } + + TStatusCallbackPair Fallback() const { auto callback = TAsyncTransformCallback([state = State] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { return FallbackCallback(state, input, output, ctx); - }); - return std::make_pair(TStatus::Async, NThreading::MakeFuture(callback)); - } - - TStatusCallbackPair HandleResult(const TExprNode::TPtr& input, TExprContext& ctx) { - YQL_LOG(DEBUG) << "Executing " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; - - if (State->ExternalUser) { - return Fallback(); - } - - TInstant startTime = TInstant::Now(); - - try { - auto result = TMaybeNode<TResult>(input).Cast(); - IDataProvider::TFillSettings fillSettings = NCommon::GetFillSettings(result.Ref()); - auto settings = State->Settings->WithFillSettings(fillSettings); + }); + return std::make_pair(TStatus::Async, NThreading::MakeFuture(callback)); + } + + TStatusCallbackPair HandleResult(const TExprNode::TPtr& input, TExprContext& ctx) { + YQL_LOG(DEBUG) << "Executing " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; + + if (State->ExternalUser) { + return Fallback(); + } + + TInstant startTime = TInstant::Now(); + + try { + auto result = TMaybeNode<TResult>(input).Cast(); + IDataProvider::TFillSettings fillSettings = NCommon::GetFillSettings(result.Ref()); + auto settings = State->Settings->WithFillSettings(fillSettings); if (!settings->_RowsLimitPerWrite.Get() && !settings->_AllResultsBytesLimit.Get()) { settings->_AllResultsBytesLimit = 64_MB; } - - THashMap<TString, TString> secureParams; - NCommon::FillSecureParams(result.Input().Ptr(), *State->TypeCtx, secureParams); - - auto graphParams = GatherGraphParams(result.Input().Ptr()); - bool hasGraphParams = !graphParams.empty(); - - TString type; - TVector<TString> columns; - GetResultType(&type, &columns, result.Ref(), result.Input().Ref()); - TString lambda; - bool untrustedUdfFlag; - int level; - TUploadList uploadList; - auto lambdaResult = GetLambda(&lambda, &untrustedUdfFlag, &level, &uploadList, result, ctx, hasGraphParams); - if (lambdaResult.first.Level == TStatus::Error) { + + THashMap<TString, TString> secureParams; + NCommon::FillSecureParams(result.Input().Ptr(), *State->TypeCtx, secureParams); + + auto graphParams = GatherGraphParams(result.Input().Ptr()); + bool hasGraphParams = !graphParams.empty(); + + TString type; + TVector<TString> columns; + GetResultType(&type, &columns, result.Ref(), result.Input().Ref()); + TString lambda; + bool untrustedUdfFlag; + int level; + TUploadList uploadList; + auto lambdaResult = GetLambda(&lambda, &untrustedUdfFlag, &level, &uploadList, result, ctx, hasGraphParams); + if (lambdaResult.first.Level == TStatus::Error) { if (State->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || State->TypeCtx->ForceDq) { return SyncError(); } - return Fallback(); - } - if (lambdaResult.first.Level != TStatus::Ok) { - return lambdaResult; - } + return Fallback(); + } + if (lambdaResult.first.Level != TStatus::Ok) { + return lambdaResult; + } THashMap<ui32, ui32> allPublicIds; bool hasStageError = false; @@ -674,15 +674,15 @@ private: IDqGateway::TDqProgressWriter progressWriter = MakeDqProgressWriter(allPublicIds); - auto executionPlanner = THolder<IDqsExecutionPlanner>(new TDqsSingleExecutionPlanner(lambda, NActors::TActorId(), NActors::TActorId(1, 0, 1, 0), State->FunctionRegistry, result.Input().Ref().GetTypeAnn())); - auto& tasks = executionPlanner->GetTasks(); - Yql::DqsProto::TTaskMeta taskMeta; - tasks[0].MutableMeta()->UnpackTo(&taskMeta); - for (const auto& file : uploadList) { - *taskMeta.AddFiles() = file; - } - tasks[0].MutableMeta()->PackFrom(taskMeta); - + auto executionPlanner = THolder<IDqsExecutionPlanner>(new TDqsSingleExecutionPlanner(lambda, NActors::TActorId(), NActors::TActorId(1, 0, 1, 0), State->FunctionRegistry, result.Input().Ref().GetTypeAnn())); + auto& tasks = executionPlanner->GetTasks(); + Yql::DqsProto::TTaskMeta taskMeta; + tasks[0].MutableMeta()->UnpackTo(&taskMeta); + for (const auto& file : uploadList) { + *taskMeta.AddFiles() = file; + } + tasks[0].MutableMeta()->PackFrom(taskMeta); + bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false); if (enableFullResultWrite) { const auto type = result.Input().Ref().GetTypeAnn(); @@ -693,162 +693,162 @@ private: && integration && integration->PrepareFullResultTableParams(result.Ref(), ctx, graphParams, secureParams); settings->EnableFullResultWrite = enableFullResultWrite; - } - - // bool executeUdfLocallyIfPossible ? - bool localRun = !State->DqGateway || (!untrustedUdfFlag && !State->TypeCtx->ForceDq && !hasGraphParams); - auto future = localRun - ? TLocalExecutor(State).Execute(ctx.GetPosition(input->Pos()), lambda, columns, secureParams, fillSettings) - : State->DqGateway->ExecutePlan( + } + + // bool executeUdfLocallyIfPossible ? + bool localRun = !State->DqGateway || (!untrustedUdfFlag && !State->TypeCtx->ForceDq && !hasGraphParams); + auto future = localRun + ? TLocalExecutor(State).Execute(ctx.GetPosition(input->Pos()), lambda, columns, secureParams, fillSettings) + : State->DqGateway->ExecutePlan( State->SessionId, *executionPlanner.Get(), columns, secureParams, graphParams, settings, progressWriter, ModulesMapping, fillSettings.Discard); - - if (State->Metrics) { - State->Metrics->IncCounter("dq", localRun - ? "InMemory" - : "Remote"); - } - - FlushStatisticsToState(); - + + if (State->Metrics) { + State->Metrics->IncCounter("dq", localRun + ? "InMemory" + : "Remote"); + } + + FlushStatisticsToState(); + return WrapFutureCallback(future, [localRun, startTime, type, fillSettings, level, settings, enableFullResultWrite, columns, graphParams, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback"; - - auto duration = TInstant::Now() - startTime; + + auto duration = TInstant::Now() - startTime; if (state->Metrics) { state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds()); state->Metrics->SetCounter( - "dq", - localRun - ? "InMemoryExecutionTime" - : "RemoteExecutionTime", - duration.MilliSeconds()); - } - + "dq", + localRun + ? "InMemoryExecutionTime" + : "RemoteExecutionTime", + duration.MilliSeconds()); + } + state->Statistics[state->MetricId++] = res.Statistics; - - if (res.Fallback) { + + if (res.Fallback) { if (state->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || state->TypeCtx->ForceDq) { auto issues = TIssues{TIssue(ctx.GetPosition(input->Pos()), "Gateway Error").SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)}; issues.AddIssues(res.Issues); ctx.AssociativeIssues.emplace(input.Get(), std::move(issues)); - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); - } - - YQL_LOG(DEBUG) << "Fallback from gateway: " << NCommon::ExprToPrettyString(ctx, *input); + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); + } + + YQL_LOG(DEBUG) << "Fallback from gateway: " << NCommon::ExprToPrettyString(ctx, *input); TIssue warning(ctx.GetPosition(input->Pos()), "DQ cannot execute the query"); - warning.Severity = TSeverityIds::S_INFO; + warning.Severity = TSeverityIds::S_INFO; ctx.IssueManager.RaiseIssue(warning); - - if (res.ForceFallback) { + + if (res.ForceFallback) { state->Metrics->IncCounter("dq", "ForceFallback"); - } + } return FallbackCallback(state, input, output, ctx); - } + } - output = input; - input->SetState(TExprNode::EState::ExecutionComplete); - - TStringStream out; + output = input; + input->SetState(TExprNode::EState::ExecutionComplete); + + TStringStream out; NYson::TYsonWriter writer((IOutputStream*)&out); - writer.OnBeginMap(); - if (type) { - writer.OnKeyedItem("Type"); - writer.OnRaw(type); - } - - writer.OnKeyedItem("Data"); - auto item = NYT::NodeFromYsonString(res.Data); - for (int i = 0; i < level; ++i) { - item = item.AsList().at(0); - } - auto raw = NYT::NodeToYsonString(item); - + writer.OnBeginMap(); + if (type) { + writer.OnKeyedItem("Type"); + writer.OnRaw(type); + } + + writer.OnKeyedItem("Data"); + auto item = NYT::NodeFromYsonString(res.Data); + for (int i = 0; i < level; ++i) { + item = item.AsList().at(0); + } + auto raw = NYT::NodeToYsonString(item); + const bool truncated = res.Truncated; const ui64 rowsCount = res.RowsCount; - + if (truncated && item.IsList()) { - ui64 bytes = 0; - ui64 rows = 0; - writer.OnBeginList(); - for (auto& node : item.AsList()) { - raw = NYT::NodeToYsonString(node); - bytes += raw.size(); - rows += 1; - writer.OnListItem(); - writer.OnRaw(raw); - if (fillSettings.AllResultsBytesLimit && bytes >= *fillSettings.AllResultsBytesLimit) { - break; - } - if (fillSettings.RowsLimitPerWrite && rows >= *fillSettings.RowsLimitPerWrite) { - break; - } - } - writer.OnEndList(); + ui64 bytes = 0; + ui64 rows = 0; + writer.OnBeginList(); + for (auto& node : item.AsList()) { + raw = NYT::NodeToYsonString(node); + bytes += raw.size(); + rows += 1; + writer.OnListItem(); + writer.OnRaw(raw); + if (fillSettings.AllResultsBytesLimit && bytes >= *fillSettings.AllResultsBytesLimit) { + break; + } + if (fillSettings.RowsLimitPerWrite && rows >= *fillSettings.RowsLimitPerWrite) { + break; + } + } + writer.OnEndList(); if (enableFullResultWrite) { - writer.OnKeyedItem("Ref"); - writer.OnBeginList(); - writer.OnListItem(); + writer.OnKeyedItem("Ref"); + writer.OnBeginList(); + writer.OnListItem(); const auto integration = GetDqIntegrationForFullResTable(state); YQL_ENSURE(integration); integration->WriteFullResultTableRef(writer, columns, graphParams); - writer.OnEndList(); - } - writer.OnKeyedItem("Truncated"); - writer.OnBooleanScalar(true); - } else if (truncated) { - writer.OnRaw("[]"); - writer.OnKeyedItem("Truncated"); - writer.OnBooleanScalar(true); - } else { - writer.OnRaw(raw); - } - + writer.OnEndList(); + } + writer.OnKeyedItem("Truncated"); + writer.OnBooleanScalar(true); + } else if (truncated) { + writer.OnRaw("[]"); + writer.OnKeyedItem("Truncated"); + writer.OnBooleanScalar(true); + } else { + writer.OnRaw(raw); + } + if (rowsCount) { writer.OnKeyedItem("RowsCount"); writer.OnUint64Scalar(rowsCount); } - writer.OnEndMap(); - - input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok); - }, ""); - - } catch (...) { + writer.OnEndMap(); + + input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok); + }, ""); + + } catch (...) { ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), CurrentExceptionMessage())); - return SyncError(); - } - } - + return SyncError(); + } + } + TStatusCallbackPair FallbackWithMessage(const TExprNode& node, const TString& message, TExprContext& ctx) { - if (State->Metrics) { - State->Metrics->IncCounter("dq", "Fallback"); - } - State->Statistics[State->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1)); + if (State->Metrics) { + State->Metrics->IncCounter("dq", "Fallback"); + } + State->Statistics[State->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1)); auto issue = TIssue(ctx.GetPosition(node.Pos()), message).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING); ctx.AssociativeIssues.emplace(&node, TIssues{std::move(issue)}); - return SyncStatus(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error)); - } - - TStatusCallbackPair HandlePull(const TExprNode::TPtr& input, TExprContext& ctx) { + return SyncStatus(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error)); + } + + TStatusCallbackPair HandlePull(const TExprNode::TPtr& input, TExprContext& ctx) { YQL_CLOG(DEBUG, ProviderDq) << "Executing " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; YQL_CLOG(TRACE, ProviderDq) << "HandlePull " << NCommon::ExprToPrettyString(ctx, *input); - - TInstant startTime = TInstant::Now(); + + TInstant startTime = TInstant::Now(); auto pull = TPull(input); - + YQL_ENSURE(!TMaybeNode<TDqQuery>(pull.Input().Ptr()) || State->Settings->EnableComputeActor.Get().GetOrElse(false), "DqQuery is not supported with worker actor"); - TString type; - TVector<TString> columns; + TString type; + TVector<TString> columns; GetResultType(&type, &columns, pull.Ref(), pull.Input().Ref()); - + const bool oneGraphPerQuery = State->Settings->_OneGraphPerQuery.Get().GetOrElse(false); size_t graphsCount = 0; THashMap<ui32, ui32> allPublicIds; - THashMap<ui64, ui32> stage2publicId; + THashMap<ui64, ui32> stage2publicId; bool hasStageError = false; VisitExpr(pull.Ptr(), [&](const TExprNode::TPtr& node) { if (TResTransientBase::Match(node.Get())) @@ -865,17 +865,17 @@ private: stage2publicId[settings.LogicalId] = *publicId; } allPublicIds.emplace(*publicId, 0U); - } - } + } + } } else if (oneGraphPerQuery) { if (expr.Maybe<TDqCnResult>() || expr.Maybe<TDqQuery>()) { ++graphsCount; } - } - return true; - }); + } + return true; + }); YQL_ENSURE(!oneGraphPerQuery || graphsCount == 1, "Internal error: only one graph per query is allowed"); - + if (hasStageError) { return SyncError(); } @@ -897,55 +897,55 @@ private: if (status != TStatus::Ok) { ctx.AddError(TIssue(ctx.GetPosition(optimizedInput->Pos()), TString("Peephole optimization failed for Dq stage"))); return SyncStatus(status); - } + } YQL_CLOG(TRACE, ProviderDq) << "After PeepHole\n" << NCommon::ExprToPrettyString(ctx, *optimizedInput); - - // copy-paste { - TUserDataTable crutches = State->TypeCtx->UserDataStorageCrutches; - TUserDataTable files; - StartCounter("FreezeUsedFiles"); + + // copy-paste { + TUserDataTable crutches = State->TypeCtx->UserDataStorageCrutches; + TUserDataTable files; + StartCounter("FreezeUsedFiles"); auto filesRes = NCommon::FreezeUsedFiles(*optimizedInput, files, *State->TypeCtx, ctx, [](const TString&){return true;}, crutches); - if (filesRes.first.Level != TStatus::Ok) { - if (filesRes.first.Level != TStatus::Error) { - YQL_LOG(DEBUG) << "Freezing files for " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; - } - return filesRes; - } - FlushCounter("FreezeUsedFiles"); - // copy-paste } - - auto executionPlanner = MakeHolder<TDqsExecutionPlanner>( - State->TypeCtx, ctx, State->FunctionRegistry, + if (filesRes.first.Level != TStatus::Ok) { + if (filesRes.first.Level != TStatus::Error) { + YQL_LOG(DEBUG) << "Freezing files for " << input->Content() << " (UniqueId=" << input->UniqueId() << ")"; + } + return filesRes; + } + FlushCounter("FreezeUsedFiles"); + // copy-paste } + + auto executionPlanner = MakeHolder<TDqsExecutionPlanner>( + State->TypeCtx, ctx, State->FunctionRegistry, optimizedInput); - + // exprRoot must be DqCnResult or DqQuery - - executionPlanner->SetPublicIds(stage2publicId); - + + executionPlanner->SetPublicIds(stage2publicId); + auto settings = std::make_shared<TDqSettings>(*State->Settings); auto tasksPerStage = settings->MaxTasksPerStage.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerStage); const auto maxTasksPerOperation = State->Settings->MaxTasksPerOperation.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerOperation); - - auto maxDataSizePerJob = settings->MaxDataSizePerJob.Get().GetOrElse(TDqSettings::TDefault::MaxDataSizePerJob); - auto stagesCount = executionPlanner->StagesCount(); - - if (!executionPlanner->CanFallback()) { - settings->FallbackPolicy = State->TypeCtx->DqFallbackPolicy = "never"; - } - - bool canFallback = (settings->FallbackPolicy.Get().GetOrElse("default") != "never" && !State->TypeCtx->ForceDq); - - if (stagesCount > maxTasksPerOperation && canFallback) { - return FallbackWithMessage( + + auto maxDataSizePerJob = settings->MaxDataSizePerJob.Get().GetOrElse(TDqSettings::TDefault::MaxDataSizePerJob); + auto stagesCount = executionPlanner->StagesCount(); + + if (!executionPlanner->CanFallback()) { + settings->FallbackPolicy = State->TypeCtx->DqFallbackPolicy = "never"; + } + + bool canFallback = (settings->FallbackPolicy.Get().GetOrElse("default") != "never" && !State->TypeCtx->ForceDq); + + if (stagesCount > maxTasksPerOperation && canFallback) { + return FallbackWithMessage( pull.Ref(), - TStringBuilder() - << "Too many stages: " - << stagesCount << " > " - << maxTasksPerOperation, ctx); - } - - YQL_ENSURE(stagesCount <= maxTasksPerOperation); - + TStringBuilder() + << "Too many stages: " + << stagesCount << " > " + << maxTasksPerOperation, ctx); + } + + YQL_ENSURE(stagesCount <= maxTasksPerOperation); + try { while (executionPlanner->PlanExecution(settings, canFallback) > maxTasksPerOperation && tasksPerStage > 1) { tasksPerStage /= 2; @@ -955,66 +955,66 @@ private: } catch (const TFallbackError& err) { YQL_ENSURE(canFallback, "Unexpected TFallbackError: " << err.what()); return FallbackWithMessage(pull.Ref(), err.what(), ctx); - } - - bool fallbackFlag = false; - if (executionPlanner->MaxDataSizePerJob() > maxDataSizePerJob && canFallback) { - return FallbackWithMessage( + } + + bool fallbackFlag = false; + if (executionPlanner->MaxDataSizePerJob() > maxDataSizePerJob && canFallback) { + return FallbackWithMessage( pull.Ref(), - TStringBuilder() - << "MaxDataSizePerJob reached: " - << executionPlanner->MaxDataSizePerJob() << " > " - << maxDataSizePerJob, ctx); - } - - bool localRun = false; - auto& tasks = executionPlanner->GetTasks(); - if (tasks.size() > maxTasksPerOperation && canFallback) { - return FallbackWithMessage( + TStringBuilder() + << "MaxDataSizePerJob reached: " + << executionPlanner->MaxDataSizePerJob() << " > " + << maxDataSizePerJob, ctx); + } + + bool localRun = false; + auto& tasks = executionPlanner->GetTasks(); + if (tasks.size() > maxTasksPerOperation && canFallback) { + return FallbackWithMessage( pull.Ref(), - TStringBuilder() - << "Too many tasks: " - << tasks.size() << " > " - << maxTasksPerOperation, ctx); - } - - YQL_ENSURE(tasks.size() <= maxTasksPerOperation); - - { - TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), State->FunctionRegistry->SupportsSizedAllocators()); - TTypeEnvironment typeEnv(alloc); - for (auto& t : tasks) { - TUploadList uploadList; - TString lambda = t.GetProgram().GetRaw(); - fallbackFlag |= BuildUploadList(&uploadList, localRun, &lambda, typeEnv, files); - t.MutableProgram()->SetRaw(lambda); - - Yql::DqsProto::TTaskMeta taskMeta; + TStringBuilder() + << "Too many tasks: " + << tasks.size() << " > " + << maxTasksPerOperation, ctx); + } + + YQL_ENSURE(tasks.size() <= maxTasksPerOperation); + + { + TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), State->FunctionRegistry->SupportsSizedAllocators()); + TTypeEnvironment typeEnv(alloc); + for (auto& t : tasks) { + TUploadList uploadList; + TString lambda = t.GetProgram().GetRaw(); + fallbackFlag |= BuildUploadList(&uploadList, localRun, &lambda, typeEnv, files); + t.MutableProgram()->SetRaw(lambda); + + Yql::DqsProto::TTaskMeta taskMeta; t.MutableMeta()->UnpackTo(&taskMeta); for (const auto& file : uploadList) { - *taskMeta.AddFiles() = file; + *taskMeta.AddFiles() = file; } t.MutableMeta()->PackFrom(taskMeta); if (const auto it = allPublicIds.find(taskMeta.GetStageId()); allPublicIds.cend() != it) ++it->second; - } - } - + } + } + MarkProgressStarted(allPublicIds, State->ProgressWriter); - if (fallbackFlag) { + if (fallbackFlag) { return FallbackWithMessage(pull.Ref(), "Too big attachment", ctx); - } - + } + IDataProvider::TFillSettings fillSettings = NCommon::GetFillSettings(pull.Ref()); - settings = settings->WithFillSettings(fillSettings); - - if (const auto optLLVM = State->TypeCtx->OptLLVM) { + settings = settings->WithFillSettings(fillSettings); + + if (const auto optLLVM = State->TypeCtx->OptLLVM) { settings->OptLLVM = *optLLVM; - } - + } + auto graphParams = GatherGraphParams(optimizedInput); - + bool ref = NCommon::HasResOrPullOption(pull.Ref(), "ref"); bool autoRef = NCommon::HasResOrPullOption(pull.Ref(), "autoref"); @@ -1026,8 +1026,8 @@ private: && integration && integration->PrepareFullResultTableParams(pull.Ref(), ctx, graphParams, secureParams); settings->EnableFullResultWrite = enableFullResultWrite; - } - + } + if (ref) { if (!enableFullResultWrite) { return FallbackWithMessage(pull.Ref(), @@ -1050,127 +1050,127 @@ private: }); executionPlanner.Destroy(); - int level = 0; - - // TODO: remove copy-paste + int level = 0; + + // TODO: remove copy-paste return WrapFutureCallback(future, [settings, startTime, localRun, type, fillSettings, level, graphParams, columns, enableFullResultWrite, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback"; - - auto duration = TInstant::Now() - startTime; + + auto duration = TInstant::Now() - startTime; if (state->Metrics) { state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds()); state->Metrics->SetCounter( - "dq", - localRun - ? "InMemoryExecutionTime" - : "RemoteExecutionTime", - duration.MilliSeconds()); - } - - state->Statistics[state->MetricId++] = res.Statistics; - - if (res.Fallback) { + "dq", + localRun + ? "InMemoryExecutionTime" + : "RemoteExecutionTime", + duration.MilliSeconds()); + } + + state->Statistics[state->MetricId++] = res.Statistics; + + if (res.Fallback) { if (state->Metrics) { state->Metrics->IncCounter("dq", "Fallback"); - } + } state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1)); - // never fallback will be captured in yql_facade + // never fallback will be captured in yql_facade auto issues = TIssues{TIssue(ctx.GetPosition(input->Pos()), "Gateway Error").SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)}; issues.AddIssues(res.Issues); ctx.AssociativeIssues.emplace(input.Get(), std::move(issues)); - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); - } - - output = input; - input->SetState(TExprNode::EState::ExecutionComplete); - - TStringStream out; + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); + } + + output = input; + input->SetState(TExprNode::EState::ExecutionComplete); + + TStringStream out; NYson::TYsonWriter writer((IOutputStream*)&out, NCommon::GetYsonFormat(fillSettings), ::NYson::EYsonType::Node, false); - writer.OnBeginMap(); - if (type) { - writer.OnKeyedItem("Type"); - writer.OnRaw(type); - } - - writer.OnKeyedItem("Data"); - auto item = NYT::NodeFromYsonString(res.Data); - for (int i = 0; i < level; ++i) { - item = item.AsList().at(0); - } - auto raw = NYT::NodeToYsonString(item); - - TString trStr = ""; + writer.OnBeginMap(); + if (type) { + writer.OnKeyedItem("Type"); + writer.OnRaw(type); + } + + writer.OnKeyedItem("Data"); + auto item = NYT::NodeFromYsonString(res.Data); + for (int i = 0; i < level; ++i) { + item = item.AsList().at(0); + } + auto raw = NYT::NodeToYsonString(item); + + TString trStr = ""; const bool truncated = res.Truncated; const ui64 rowsCount = res.RowsCount; - + if (truncated && !state->TypeCtx->ForceDq && !enableFullResultWrite) { auto issue = TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "DQ cannot execute the query. Cause: " << "too big result " << trStr).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_INFO); - bool error = settings->FallbackPolicy.Get().GetOrElse("default") == "never"; - for (const auto& i : res.Issues) { - TIssuePtr subIssue = new TIssue(i); - if (error && subIssue->Severity == TSeverityIds::S_WARNING) { - subIssue->Severity = TSeverityIds::S_ERROR; - } - issue.AddSubIssue(subIssue); - } - - if (error) { - issue.Message = "Too big result " + trStr; - issue.Severity = TSeverityIds::S_ERROR; - } + bool error = settings->FallbackPolicy.Get().GetOrElse("default") == "never"; + for (const auto& i : res.Issues) { + TIssuePtr subIssue = new TIssue(i); + if (error && subIssue->Severity == TSeverityIds::S_WARNING) { + subIssue->Severity = TSeverityIds::S_ERROR; + } + issue.AddSubIssue(subIssue); + } + + if (error) { + issue.Message = "Too big result " + trStr; + issue.Severity = TSeverityIds::S_ERROR; + } ctx.IssueManager.RaiseIssue(issue); - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); - } - - if (truncated) { - // TODO: - ui64 bytes = 0; - ui64 rows = 0; - writer.OnBeginList(); - for (auto& node : item.AsList()) { - raw = NYT::NodeToYsonString(node); - bytes += raw.size(); - rows += 1; - writer.OnListItem(); - writer.OnRaw(raw); - if (fillSettings.AllResultsBytesLimit && bytes >= *fillSettings.AllResultsBytesLimit) { - break; - } - if (fillSettings.RowsLimitPerWrite && rows >= *fillSettings.RowsLimitPerWrite) { - break; - } - } - writer.OnEndList(); - + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error); + } + + if (truncated) { + // TODO: + ui64 bytes = 0; + ui64 rows = 0; + writer.OnBeginList(); + for (auto& node : item.AsList()) { + raw = NYT::NodeToYsonString(node); + bytes += raw.size(); + rows += 1; + writer.OnListItem(); + writer.OnRaw(raw); + if (fillSettings.AllResultsBytesLimit && bytes >= *fillSettings.AllResultsBytesLimit) { + break; + } + if (fillSettings.RowsLimitPerWrite && rows >= *fillSettings.RowsLimitPerWrite) { + break; + } + } + writer.OnEndList(); + if (enableFullResultWrite) { - writer.OnKeyedItem("Ref"); - writer.OnBeginList(); - writer.OnListItem(); + writer.OnKeyedItem("Ref"); + writer.OnBeginList(); + writer.OnListItem(); const auto integration = GetDqIntegrationForFullResTable(state); YQL_ENSURE(integration); integration->WriteFullResultTableRef(writer, columns, graphParams); - writer.OnEndList(); - } - - writer.OnKeyedItem("Truncated"); - writer.OnBooleanScalar(true); - } else { - writer.OnRaw(raw); - } - + writer.OnEndList(); + } + + writer.OnKeyedItem("Truncated"); + writer.OnBooleanScalar(true); + } else { + writer.OnRaw(raw); + } + if (rowsCount) { writer.OnKeyedItem("RowsCount"); writer.OnUint64Scalar(rowsCount); } - writer.OnEndMap(); - + writer.OnEndMap(); + ctx.IssueManager.RaiseIssues(res.Issues); - input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok); - }, ""); - } - + input->SetResult(ctx.NewAtom(input->Pos(), out.Str())); + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok); + }, ""); + } + IDqGateway::TDqProgressWriter MakeDqProgressWriter(const THashMap<ui32, ui32>& allPublicIds) const { IDqGateway::TDqProgressWriter dqProgressWriter = [progressWriter = State->ProgressWriter, allPublicIds](const TString& stage) { for (const auto& publicId : allPublicIds) { @@ -1208,16 +1208,16 @@ private: } } - void FlushStatisticsToState() { - TOperationStatistics statistics; - FlushCounters(statistics); - - TGuard<TMutex> lock(State->Mutex); - if (!statistics.Entries.empty()) { - State->Statistics[State->MetricId++] = statistics; - } - } - + void FlushStatisticsToState() { + TOperationStatistics statistics; + FlushCounters(statistics); + + TGuard<TMutex> lock(State->Mutex); + if (!statistics.Entries.empty()) { + State->Statistics[State->MetricId++] = statistics; + } + } + THashMap<TString, TString> GatherGraphParams(const TExprNode::TPtr& root) { THashMap<TString, TString> params; VisitExpr(root, [&](const TExprNode::TPtr& node) -> bool { @@ -1254,16 +1254,16 @@ private: return nullptr; } - TDqStatePtr State; - THolder<IGraphTransformer> DqTypeAnnotationTransformer; - mutable THashMap<TString, TFileLinkPtr> FileLinks; - mutable THashMap<TString, TString> ModulesMapping; -}; - + TDqStatePtr State; + THolder<IGraphTransformer> DqTypeAnnotationTransformer; + mutable THashMap<TString, TFileLinkPtr> FileLinks; + mutable THashMap<TString, TString> ModulesMapping; +}; + +} + +IGraphTransformer* CreateInMemoryExecTransformer(const TDqStatePtr& state) { + return new TInMemoryExecTransformer(state); } -IGraphTransformer* CreateInMemoryExecTransformer(const TDqStatePtr& state) { - return new TInMemoryExecTransformer(state); -} - -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h index 8851ad2b82..e6fdd644a1 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_data_provider.h> - + #include <util/generic/ptr.h> -namespace NYql { - struct TDqState; +namespace NYql { + struct TDqState; using TDqStatePtr = TIntrusivePtr<TDqState>; - - IGraphTransformer* CreateInMemoryExecTransformer(const TDqStatePtr& state); -} // namespace NYql + + IGraphTransformer* CreateInMemoryExecTransformer(const TDqStatePtr& state); +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/ya.make b/ydb/library/yql/providers/dq/provider/ya.make index 2265324957..bdb3547266 100644 --- a/ydb/library/yql/providers/dq/provider/ya.make +++ b/ydb/library/yql/providers/dq/provider/ya.make @@ -1,27 +1,27 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( +LIBRARY() + +OWNER(g:yql) + +SRCS( yql_dq_control.cpp yql_dq_control.h yql_dq_datasink_type_ann.cpp yql_dq_datasink_type_ann.h yql_dq_datasource_type_ann.cpp yql_dq_datasource_type_ann.h - yql_dq_gateway.cpp - yql_dq_gateway.h - yql_dq_provider.cpp - yql_dq_provider.h - yql_dq_datasink.cpp - yql_dq_datasink.h - yql_dq_datasource.cpp - yql_dq_datasource.h + yql_dq_gateway.cpp + yql_dq_gateway.h + yql_dq_provider.cpp + yql_dq_provider.h + yql_dq_datasink.cpp + yql_dq_datasink.h + yql_dq_datasource.cpp + yql_dq_datasource.h yql_dq_recapture.cpp yql_dq_recapture.h -) - -PEERDIR( +) + +PEERDIR( library/cpp/grpc/client library/cpp/threading/task_scheduler ydb/public/lib/yson_value @@ -43,8 +43,8 @@ PEERDIR( ydb/library/yql/providers/dq/opt ydb/library/yql/providers/dq/planner ydb/library/yql/providers/result/expr_nodes -) - +) + YQL_LAST_ABI_VERSION() -END() +END() diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp index c6adf3a089..1a0053350a 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_control.cpp @@ -1,62 +1,62 @@ #include "yql_dq_control.h" - + #include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h> #include <ydb/library/yql/providers/dq/config/config.pb.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> - + #include <ydb/public/lib/yson_value/ydb_yson_value.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> - + #include <library/cpp/svnversion/svnversion.h> -#include <util/memory/blob.h> -#include <util/string/builder.h> -#include <util/system/file.h> - -namespace NYql { +#include <util/memory/blob.h> +#include <util/string/builder.h> +#include <util/system/file.h> + +namespace NYql { using TFileResource = Yql::DqsProto::TFile; -const TString DqStrippedSuffied = ".s"; - +const TString DqStrippedSuffied = ".s"; + class TDqControl : public IDqControl { -public: +public: TDqControl(const NGrpc::TGRpcClientConfig &grpcConf, int threads, const TVector<TFileResource> &files) : GrpcClient(threads) , Service(GrpcClient.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(grpcConf)) , Files(files) - { } - + { } + // after call, forking process in not allowed bool IsReady(const TMap<TString, TString>& additinalFiles) override { Yql::DqsProto::IsReadyRequest request; for (const auto& file : Files) { *request.AddFiles() = file; - } - - for (const auto& [path, objectId] : additinalFiles){ + } + + for (const auto& [path, objectId] : additinalFiles){ TFileResource r; r.SetLocalPath(path); - r.SetObjectType(Yql::DqsProto::TFile::EUDF_FILE); - r.SetObjectId(objectId); - r.SetSize(TFile(path, OpenExisting | RdOnly).GetLength()); + r.SetObjectType(Yql::DqsProto::TFile::EUDF_FILE); + r.SetObjectId(objectId); + r.SetSize(TFile(path, OpenExisting | RdOnly).GetLength()); *request.AddFiles() = r; } auto promise = NThreading::NewPromise<bool>(); auto callback = [promise](NGrpc::TGrpcStatus&& status, Yql::DqsProto::IsReadyResponse&& resp) mutable { Y_UNUSED(resp); - + promise.SetValue(status.Ok() && resp.GetIsReady()); - }; - - NGrpc::TCallMeta meta; - meta.Timeout = TDuration::Seconds(1); + }; + + NGrpc::TCallMeta meta; + meta.Timeout = TDuration::Seconds(1); Service->DoRequest<Yql::DqsProto::IsReadyRequest, Yql::DqsProto::IsReadyResponse>( - request, callback, &Yql::DqsProto::DqService::Stub::AsyncIsReady, meta); + request, callback, &Yql::DqsProto::DqService::Stub::AsyncIsReady, meta); try { return promise.GetFuture().GetValueSync(); @@ -64,9 +64,9 @@ public: YQL_LOG(INFO) << "DqControl IsReady Exception " << CurrentExceptionMessage(); return false; } - } - -private: + } + +private: NGrpc::TGRpcClientLow GrpcClient; std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> Service; const TVector<TFileResource> &Files; @@ -82,44 +82,44 @@ public: int threads, const TMap<TString, TString>& udfs, const TString& vanillaLitePath, - const TString& vanillaLiteMd5, - const THashSet<TString> &filter, - bool enableStrip, - const TFileStoragePtr& fileStorage + const TString& vanillaLiteMd5, + const THashSet<TString> &filter, + bool enableStrip, + const TFileStoragePtr& fileStorage ) : Threads(threads) , GrpcConf(TStringBuilder() << host << ":" << port) , IndexedUdfFilter(filter) - , EnableStrip(enableStrip) - , FileStorage(fileStorage) + , EnableStrip(enableStrip) + , FileStorage(fileStorage) { if (!vanillaLitePath.empty()) { - TString path = vanillaLitePath; - TString objectId = GetProgramCommitId(); - - TString newPath, newObjectId; - std::tie(newPath, newObjectId) = GetPathAndObjectId(path, objectId, vanillaLiteMd5); - + TString path = vanillaLitePath; + TString objectId = GetProgramCommitId(); + + TString newPath, newObjectId; + std::tie(newPath, newObjectId) = GetPathAndObjectId(path, objectId, vanillaLiteMd5); + TFileResource vanillaLite; - vanillaLite.SetLocalPath(newPath); + vanillaLite.SetLocalPath(newPath); vanillaLite.SetName(vanillaLitePath.substr(vanillaLitePath.rfind('/') + 1)); - vanillaLite.SetObjectType(Yql::DqsProto::TFile::EEXE_FILE); - vanillaLite.SetObjectId(newObjectId); - vanillaLite.SetSize(TFile(newPath, OpenExisting | RdOnly).GetLength()); + vanillaLite.SetObjectType(Yql::DqsProto::TFile::EEXE_FILE); + vanillaLite.SetObjectId(newObjectId); + vanillaLite.SetSize(TFile(newPath, OpenExisting | RdOnly).GetLength()); Files.push_back(vanillaLite); } - for (const auto& [path, objectId] : udfs){ - YQL_LOG(DEBUG) << "DQ control, adding file: " << path << " with objectId " << objectId; - TString newPath, newObjectId; - std::tie(newPath, newObjectId) = GetPathAndObjectId(path, objectId, objectId); - - YQL_LOG(DEBUG) << "DQ control, rewrite path/objectId: " << newPath << ", " << newObjectId; + for (const auto& [path, objectId] : udfs){ + YQL_LOG(DEBUG) << "DQ control, adding file: " << path << " with objectId " << objectId; + TString newPath, newObjectId; + std::tie(newPath, newObjectId) = GetPathAndObjectId(path, objectId, objectId); + + YQL_LOG(DEBUG) << "DQ control, rewrite path/objectId: " << newPath << ", " << newObjectId; TFileResource r; - r.SetLocalPath(newPath); - r.SetObjectType(Yql::DqsProto::TFile::EUDF_FILE); - r.SetObjectId(newObjectId); - r.SetSize(TFile(newPath, OpenExisting | RdOnly).GetLength()); + r.SetLocalPath(newPath); + r.SetObjectType(Yql::DqsProto::TFile::EUDF_FILE); + r.SetObjectId(newObjectId); + r.SetSize(TFile(newPath, OpenExisting | RdOnly).GetLength()); Files.push_back(r); } } @@ -132,46 +132,46 @@ public: return IndexedUdfFilter; } - bool StripEnabled() const override { - return EnableStrip; - } - + bool StripEnabled() const override { + return EnableStrip; + } + private: - std::tuple<TString, TString> GetPathAndObjectId(const TString& path, const TString& objectId, const TString& md5 = {}) { - if (!EnableStrip) { - return std::make_tuple(path, objectId); - } - - TFileLinkPtr& fileLink = FileLinks[objectId]; - if (!fileLink) { - fileLink = FileStorage->PutFileStripped(path, md5); - } - - return std::make_tuple(fileLink->GetPath(), objectId + DqStrippedSuffied); - } - + std::tuple<TString, TString> GetPathAndObjectId(const TString& path, const TString& objectId, const TString& md5 = {}) { + if (!EnableStrip) { + return std::make_tuple(path, objectId); + } + + TFileLinkPtr& fileLink = FileLinks[objectId]; + if (!fileLink) { + fileLink = FileStorage->PutFileStripped(path, md5); + } + + return std::make_tuple(fileLink->GetPath(), objectId + DqStrippedSuffied); + } + int Threads; TVector<TFileResource> Files; - NGrpc::TGRpcClientConfig GrpcConf; + NGrpc::TGRpcClientConfig GrpcConf; THashSet<TString> IndexedUdfFilter; - THashMap<TString, TFileLinkPtr> FileLinks; - bool EnableStrip; - const TFileStoragePtr FileStorage; -}; - -IDqControlFactoryPtr CreateDqControlFactory(const NProto::TDqConfig& config, const TMap<TString, TString>& udfs, const TFileStoragePtr& fileStorage) { + THashMap<TString, TFileLinkPtr> FileLinks; + bool EnableStrip; + const TFileStoragePtr FileStorage; +}; + +IDqControlFactoryPtr CreateDqControlFactory(const NProto::TDqConfig& config, const TMap<TString, TString>& udfs, const TFileStoragePtr& fileStorage) { THashSet<TString> indexedUdfFilter(config.GetControl().GetIndexedUdfsToWarmup().begin(), config.GetControl().GetIndexedUdfsToWarmup().end()); - return new TDqControlFactory( - "localhost", - config.GetPort(), - 2, - udfs, - config.GetYtBackends()[0].GetVanillaJob(), - config.GetYtBackends()[0].GetVanillaJobMd5(), - indexedUdfFilter, - config.GetControl().GetEnableStrip(), - fileStorage - ); -} - -} // namespace NYql + return new TDqControlFactory( + "localhost", + config.GetPort(), + 2, + udfs, + config.GetYtBackends()[0].GetVanillaJob(), + config.GetYtBackends()[0].GetVanillaJobMd5(), + indexedUdfFilter, + config.GetControl().GetEnableStrip(), + fileStorage + ); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_control.h b/ydb/library/yql/providers/dq/provider/yql_dq_control.h index 2a1ad8bbcd..d6b58eacd7 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_control.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_control.h @@ -1,38 +1,38 @@ -#pragma once - -#include <util/generic/ptr.h> +#pragma once + +#include <util/generic/ptr.h> #include <util/generic/map.h> - + #include <ydb/library/yql/core/file_storage/file_storage.h> - -namespace NYql { - + +namespace NYql { + namespace NProto { class TDqConfig; } class IDqControl : public TThrRefBase { -public: +public: using TPtr = TIntrusivePtr<IDqControl>; - + virtual bool IsReady(const TMap<TString, TString>& udfs = TMap<TString, TString>())= 0; }; - + using IDqControlPtr = TIntrusivePtr<IDqControl>; - + class IDqControlFactory : public TThrRefBase { public: using TPtr = TIntrusivePtr<IDqControlFactory>; - + virtual IDqControlPtr GetControl() = 0; virtual const THashSet<TString>& GetIndexedUdfFilter() = 0; - virtual bool StripEnabled() const = 0; + virtual bool StripEnabled() const = 0; }; using IDqControlFactoryPtr = TIntrusivePtr<IDqControlFactory>; -IDqControlFactoryPtr CreateDqControlFactory(const NProto::TDqConfig& config, const TMap<TString, TString>& udfs, const TFileStoragePtr& fileStorage); +IDqControlFactoryPtr CreateDqControlFactory(const NProto::TDqConfig& config, const TMap<TString, TString>& udfs, const TFileStoragePtr& fileStorage); + +extern const TString DqStrippedSuffied; -extern const TString DqStrippedSuffied; - -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp index 3aae02e072..a8b5e598a0 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp @@ -1,8 +1,8 @@ -#include "yql_dq_datasink.h" -#include "yql_dq_state.h" +#include "yql_dq_datasink.h" +#include "yql_dq_state.h" #include "yql_dq_datasink_type_ann.h" #include "yql_dq_recapture.h" - + #include <ydb/library/yql/providers/dq/opt/logical_optimize.h> #include <ydb/library/yql/providers/dq/opt/physical_optimize.h> #include <ydb/library/yql/providers/dq/opt/dqs_opt.h> @@ -15,142 +15,142 @@ #include <ydb/library/yql/providers/common/transform/yql_lazy_init.h> #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/core/yql_expr_type_annotation.h> - + #include <ydb/library/yql/core/services/yql_transform_pipeline.h> #include <ydb/library/yql/utils/log/log.h> - -namespace NYql { - + +namespace NYql { + using namespace NNodes; class TDqDataProviderSink: public TDataProviderBase { -public: +public: TDqDataProviderSink(const TDqStatePtr& state) - : State(state) + : State(state) , LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); }) , PhyOptTransformer([] () { return CreateDqsPhyOptTransformer(/*TODO: State->TypeCtx*/nullptr); }) , PhysicalFinalizingTransformer([] () { return CreateDqsFinalizingOptTransformer(); }) , TypeAnnotationTransformer([state] () { return CreateDqsDataSinkTypeAnnotationTransformer(state->TypeCtx); }) , RecaptureTransformer([state] () { return CreateDqsRecaptureTransformer(state); }) - { } - + { } + bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) override { - auto statistics = [&]() { - TGuard<TMutex> lock(State->Mutex); - return State->Statistics; - } (); - - if (statistics.empty()) { - return false; - } - - TOperationStatistics taskRunner; - THashMap<TString, std::tuple<i64, i64, i64, TMaybe<i64>>> total; // sum, count, max, min - - for (auto& opStatistics : statistics) { - TOperationStatistics newStat; - for (auto& el : opStatistics.second.Entries) { - - if (el.Value) { - continue; - } - - if (el.Name.StartsWith("TaskRunner")) { - taskRunner.Entries.push_back(el); - } else { - newStat.Entries.push_back(el); - } - - auto& totalEntry = total[el.Name]; - if (auto val = el.Sum) { - std::get<0>(totalEntry) += *val; - } - if (auto val = el.Count) { - std::get<1>(totalEntry) += *val; - } - if (auto val = el.Max) { - std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); - } - if (auto val = el.Min) { - std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); - } - } - opStatistics.second = newStat; - } - - // NCommon::WriteStatistics(writer, totalOnly, statistics); - - writer.OnBeginMap(); - writer.OnKeyedItem("All"); - NCommon::WriteStatistics(writer, totalOnly, statistics); - - THashMap<TString, TOperationStatistics> taskRunnerStage; - THashMap<TString, TOperationStatistics> taskRunnerInput; - THashMap<TString, TOperationStatistics> taskRunnerOutput; - - for (const auto& entry : taskRunner.Entries) { - TString prefix, name; - std::map<TString, TString> labels; + auto statistics = [&]() { + TGuard<TMutex> lock(State->Mutex); + return State->Statistics; + } (); + + if (statistics.empty()) { + return false; + } + + TOperationStatistics taskRunner; + THashMap<TString, std::tuple<i64, i64, i64, TMaybe<i64>>> total; // sum, count, max, min + + for (auto& opStatistics : statistics) { + TOperationStatistics newStat; + for (auto& el : opStatistics.second.Entries) { + + if (el.Value) { + continue; + } + + if (el.Name.StartsWith("TaskRunner")) { + taskRunner.Entries.push_back(el); + } else { + newStat.Entries.push_back(el); + } + + auto& totalEntry = total[el.Name]; + if (auto val = el.Sum) { + std::get<0>(totalEntry) += *val; + } + if (auto val = el.Count) { + std::get<1>(totalEntry) += *val; + } + if (auto val = el.Max) { + std::get<2>(totalEntry) = Max<i64>(*val, std::get<2>(totalEntry)); + } + if (auto val = el.Min) { + std::get<3>(totalEntry) = Min<i64>(*val, std::get<3>(totalEntry).GetOrElse(Max<i64>())); + } + } + opStatistics.second = newStat; + } + + // NCommon::WriteStatistics(writer, totalOnly, statistics); + + writer.OnBeginMap(); + writer.OnKeyedItem("All"); + NCommon::WriteStatistics(writer, totalOnly, statistics); + + THashMap<TString, TOperationStatistics> taskRunnerStage; + THashMap<TString, TOperationStatistics> taskRunnerInput; + THashMap<TString, TOperationStatistics> taskRunnerOutput; + + for (const auto& entry : taskRunner.Entries) { + TString prefix, name; + std::map<TString, TString> labels; if (!NCommon::ParseCounterName(&prefix, &labels, &name, entry.Name)) { - continue; - } - auto maybeInput = labels.find("Input"); - auto maybeOutput = labels.find("Output"); - auto maybeStage = labels.find("Stage"); - if (maybeStage == labels.end()) { - maybeStage = labels.find("Task"); - } - if (maybeStage == labels.end()) { - continue; - } - - if (maybeInput != labels.end()) { - auto newEntry = entry; newEntry.Name = name; - taskRunnerInput[maybeStage->second].Entries.push_back(newEntry); - } - if (maybeOutput != labels.end()) { - auto newEntry = entry; newEntry.Name = name; - taskRunnerOutput[maybeStage->second].Entries.push_back(newEntry); - } - if (maybeInput == labels.end() && maybeOutput == labels.end()) { - auto newEntry = entry; newEntry.Name = name; - taskRunnerStage[maybeStage->second].Entries.push_back(newEntry); - } - } - - writer.OnKeyedItem("TaskRunner"); - - { - writer.OnBeginMap(); - for (const auto& [stageId, stat] : taskRunnerStage) { - auto& inputStat = taskRunnerInput[stageId]; - auto& outputStat = taskRunnerInput[stageId]; - - writer.OnKeyedItem("Stage=" + stageId); - { - writer.OnBeginMap(); - - writer.OnKeyedItem("Input"); - NCommon::WriteStatistics(writer, totalOnly, {{0, inputStat}}); - - writer.OnKeyedItem("Output"); - NCommon::WriteStatistics(writer, totalOnly, {{0, outputStat}}); - - writer.OnKeyedItem("Task"); - NCommon::WriteStatistics(writer, totalOnly, {{0, stat}}); - - writer.OnEndMap(); - } - } - - writer.OnEndMap(); - } - - writer.OnEndMap(); - - return true; - } - + continue; + } + auto maybeInput = labels.find("Input"); + auto maybeOutput = labels.find("Output"); + auto maybeStage = labels.find("Stage"); + if (maybeStage == labels.end()) { + maybeStage = labels.find("Task"); + } + if (maybeStage == labels.end()) { + continue; + } + + if (maybeInput != labels.end()) { + auto newEntry = entry; newEntry.Name = name; + taskRunnerInput[maybeStage->second].Entries.push_back(newEntry); + } + if (maybeOutput != labels.end()) { + auto newEntry = entry; newEntry.Name = name; + taskRunnerOutput[maybeStage->second].Entries.push_back(newEntry); + } + if (maybeInput == labels.end() && maybeOutput == labels.end()) { + auto newEntry = entry; newEntry.Name = name; + taskRunnerStage[maybeStage->second].Entries.push_back(newEntry); + } + } + + writer.OnKeyedItem("TaskRunner"); + + { + writer.OnBeginMap(); + for (const auto& [stageId, stat] : taskRunnerStage) { + auto& inputStat = taskRunnerInput[stageId]; + auto& outputStat = taskRunnerInput[stageId]; + + writer.OnKeyedItem("Stage=" + stageId); + { + writer.OnBeginMap(); + + writer.OnKeyedItem("Input"); + NCommon::WriteStatistics(writer, totalOnly, {{0, inputStat}}); + + writer.OnKeyedItem("Output"); + NCommon::WriteStatistics(writer, totalOnly, {{0, outputStat}}); + + writer.OnKeyedItem("Task"); + NCommon::WriteStatistics(writer, totalOnly, {{0, stat}}); + + writer.OnEndMap(); + } + } + + writer.OnEndMap(); + } + + writer.OnEndMap(); + + return true; + } + bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { if (node.IsCallable(TCoDataSink::CallableName())) { if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) { @@ -181,31 +181,31 @@ public: return TypeAnnotationTransformer->CanParse(node); } - IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { - Y_UNUSED(instantOnly); - return *TypeAnnotationTransformer; - } - + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { + Y_UNUSED(instantOnly); + return *TypeAnnotationTransformer; + } + IGraphTransformer& GetRecaptureOptProposalTransformer() override { return *RecaptureTransformer; } - IGraphTransformer& GetLogicalOptProposalTransformer() override { + IGraphTransformer& GetLogicalOptProposalTransformer() override { return *LogOptTransformer; - } - - IGraphTransformer& GetPhysicalOptProposalTransformer() override { + } + + IGraphTransformer& GetPhysicalOptProposalTransformer() override { return *PhyOptTransformer; - } + } - IGraphTransformer& GetPhysicalFinalizingTransformer() override { + IGraphTransformer& GetPhysicalFinalizingTransformer() override { return *PhysicalFinalizingTransformer; - } + } - TStringBuf GetName() const override { + TStringBuf GetName() const override { return DqProviderName; - } - + } + bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { Y_UNUSED(compact); @@ -265,17 +265,17 @@ public: } } - TDqStatePtr State; - - TLazyInitHolder<IGraphTransformer> LogOptTransformer; - TLazyInitHolder<IGraphTransformer> PhyOptTransformer; + TDqStatePtr State; + + TLazyInitHolder<IGraphTransformer> LogOptTransformer; + TLazyInitHolder<IGraphTransformer> PhyOptTransformer; TLazyInitHolder<IGraphTransformer> PhysicalFinalizingTransformer; TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer; TLazyInitHolder<IGraphTransformer> RecaptureTransformer; -}; - +}; + TIntrusivePtr<IDataProvider> CreateDqDataSink(const TDqStatePtr& state) { return new TDqDataProviderSink(state); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h index be942b3061..d9849824a7 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_data_provider.h> - -namespace NYql { - struct TDqState; + +namespace NYql { + struct TDqState; using TDqStatePtr = TIntrusivePtr<TDqState>; - + TIntrusivePtr<IDataProvider> CreateDqDataSink(const TDqStatePtr& state); -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp index c14d55413e..aea55435de 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp @@ -1,7 +1,7 @@ -#include "yql_dq_datasource.h" +#include "yql_dq_datasource.h" #include "yql_dq_datasource_type_ann.h" -#include "yql_dq_state.h" - +#include "yql_dq_state.h" + #include <ydb/library/yql/providers/common/config/yql_configuration_transformer.h> #include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> @@ -9,70 +9,70 @@ #include <ydb/library/yql/providers/common/transform/yql_exec.h> #include <ydb/library/yql/providers/common/transform/yql_lazy_init.h> #include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h> - + #include <ydb/library/yql/providers/dq/opt/dqs_opt.h> #include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h> #include <ydb/library/yql/providers/dq/common/yql_dq_common.h> #include <ydb/library/yql/providers/dq/planner/execution_planner.h> - + #include <ydb/library/yql/dq/opt/dq_opt_build.h> #include <ydb/library/yql/dq/opt/dq_opt.h> - + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/core/services/yql_transform_pipeline.h> - + #include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h> #include <ydb/library/yql/core/type_ann/type_ann_expr.h> #include <ydb/library/yql/core/yql_type_annotation.h> #include <ydb/library/yql/core/yql_type_helpers.h> #include <ydb/library/yql/core/yql_graph_transformer.h> - -namespace NYql { - -using namespace NCommon; -using namespace NKikimr::NMiniKQL; -using namespace NNodes; -using namespace NDq; - + +namespace NYql { + +using namespace NCommon; +using namespace NKikimr::NMiniKQL; +using namespace NNodes; +using namespace NDq; + class TDqDataProviderSource: public TDataProviderBase { -public: +public: TDqDataProviderSource(const TDqStatePtr& state, TExecTransformerFactory execTransformerFactory) - : State(state) - , ConfigurationTransformer([this]() { + : State(state) + , ConfigurationTransformer([this]() { return MakeHolder<NCommon::TProviderConfigurationTransformer>(State->Settings, *State->TypeCtx, TString{DqProviderName}); - }) + }) , ExecTransformer([this, execTransformerFactory] () { return THolder<IGraphTransformer>(execTransformerFactory(State)); }) , TypeAnnotationTransformer([] () { return CreateDqsDataSourceTypeAnnotationTransformer(); }) - { } - - TStringBuf GetName() const override { + { } + + TStringBuf GetName() const override { return DqProviderName; - } - - IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { - Y_UNUSED(instantOnly); - return *TypeAnnotationTransformer; - } - - IGraphTransformer& GetConfigurationTransformer() override { - return *ConfigurationTransformer; - } - + } + + IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override { + Y_UNUSED(instantOnly); + return *TypeAnnotationTransformer; + } + + IGraphTransformer& GetConfigurationTransformer() override { + return *ConfigurationTransformer; + } + TExprNode::TPtr OptimizePull(const TExprNode::TPtr& node, const TFillSettings& fillSettings, TExprContext& ctx, - IOptimizationContext& optCtx) override - { - Y_UNUSED(optCtx); - Y_UNUSED(fillSettings); - + IOptimizationContext& optCtx) override + { + Y_UNUSED(optCtx); + Y_UNUSED(fillSettings); + if (TDqCnResult::Match(node.Get())) { return node; } - + if (!TDqCnUnionAll::Match(node.Get())) { ctx.AddError(TIssue(node->Pos(ctx), "Last connection must be union all")); return {}; } - + TExprNode::TListType worlds; VisitExpr(node, [&worlds] (const TExprNode::TPtr& item) { if (ETypeAnnotationKind::World == item->GetTypeAnn()->GetKind()) { @@ -97,16 +97,16 @@ public: .Args({"row"}) .Body("row") .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node->Pos())) + .Settings(TDqStageSettings().BuildNode(ctx, node->Pos())) .Build() .Index().Build("0") .Build() .ColumnHints() // TODO: set column hints .Build() .Done().Ptr(); - } - - bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override { + } + + bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override { if (!TDqCnUnionAll::Match(&node)) { return false; } @@ -133,78 +133,78 @@ public: return true; }); return true; - } - + } + bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override { if (node.IsCallable(TCoDataSource::CallableName())) { if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) { return false; - } - + } + if (node.Child(0)->Content() == DqProviderName) { if (node.ChildrenSize() == 2) { if (!EnsureAtom(*node.Child(1), ctx)) { return false; } - + if (node.Child(1)->Content() != "$all") { ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unexpected cluster name: " << node.Child(1)->Content())); return false; } } cluster = Nothing(); - return true; - } - } - + return true; + } + } + ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid DQ DataSource parameters")); return false; - } - - bool CanExecute(const TExprNode& node) override { + } + + bool CanExecute(const TExprNode& node) override { return TDqCnResult::Match(&node) || TDqQuery::Match(&node); - } - - bool CanParse(const TExprNode& node) override { + } + + bool CanParse(const TExprNode& node) override { return TypeAnnotationTransformer->CanParse(node); - } - - IGraphTransformer& GetCallableExecutionTransformer() override { - return *ExecTransformer; - } - - bool Initialize(TExprContext& ctx) override { - auto category = YtProviderName; - auto cred = State->TypeCtx->FindCredential(TString("default_").append(category)); - if (cred) { - if (cred->Category != category) { - ctx.AddError(TIssue({}, TStringBuilder() - << "Mismatch default credential category, expected: " << category - << ", but found: " << cred->Category)); - return false; - } - State->YtToken = cred->Content; - } - - return true; - } - - void Reset() final { - if (ExecTransformer) { - ExecTransformer->Rewind(); - TypeAnnotationTransformer->Rewind(); - } - } - -private: - TDqStatePtr State; - TLazyInitHolder<IGraphTransformer> ConfigurationTransformer; - TLazyInitHolder<IGraphTransformer> ExecTransformer; + } + + IGraphTransformer& GetCallableExecutionTransformer() override { + return *ExecTransformer; + } + + bool Initialize(TExprContext& ctx) override { + auto category = YtProviderName; + auto cred = State->TypeCtx->FindCredential(TString("default_").append(category)); + if (cred) { + if (cred->Category != category) { + ctx.AddError(TIssue({}, TStringBuilder() + << "Mismatch default credential category, expected: " << category + << ", but found: " << cred->Category)); + return false; + } + State->YtToken = cred->Content; + } + + return true; + } + + void Reset() final { + if (ExecTransformer) { + ExecTransformer->Rewind(); + TypeAnnotationTransformer->Rewind(); + } + } + +private: + TDqStatePtr State; + TLazyInitHolder<IGraphTransformer> ConfigurationTransformer; + TLazyInitHolder<IGraphTransformer> ExecTransformer; TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer; -}; - +}; + TIntrusivePtr<IDataProvider> CreateDqDataSource(const TDqStatePtr& state, TExecTransformerFactory execTransformerFactory) { return new TDqDataProviderSource(state, execTransformerFactory); -} - -} // namespace NYql +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h index dfa2cf24a5..500ec35d4a 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/core/yql_data_provider.h> - + #include <util/generic/ptr.h> -namespace NYql { - struct TDqState; +namespace NYql { + struct TDqState; using TDqStatePtr = TIntrusivePtr<TDqState>; - + using TExecTransformerFactory = std::function<IGraphTransformer*(const TDqStatePtr& state)>; TIntrusivePtr<IDataProvider> CreateDqDataSource(const TDqStatePtr& state, TExecTransformerFactory execTransformerFactory); -} // namespace NYql +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp index 2da6584621..2341b28ad4 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp @@ -1,58 +1,58 @@ -#include "yql_dq_gateway.h" - +#include "yql_dq_gateway.h" + #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/dq/api/grpc/api.grpc.pb.h> #include <ydb/library/yql/providers/dq/backtrace/backtrace.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> #include <ydb/library/yql/providers/dq/config/config.pb.h> #include <ydb/library/yql/utils/log/log.h> - + #include <ydb/public/lib/yson_value/ydb_yson_value.h> - + #include <library/cpp/grpc/client/grpc_client_low.h> #include <library/cpp/yson/node/node_io.h> -#include <library/cpp/threading/task_scheduler/task_scheduler.h> - +#include <library/cpp/threading/task_scheduler/task_scheduler.h> + #include <util/system/thread.h> - + #include <utility> -namespace NYql { - -class TDqGateway: public IDqGateway -{ - enum ETaskPriority { - PRIO_NORMAL = 1, - PRIO_RT = 2 - }; - -public: - TDqGateway(const TString& host, int port, int threads, const TString& vanillaJobPath, const TString& vanillaJobMd5, TDuration timeout = TDuration::Minutes(60)) - : GrpcConf(TStringBuilder() << host << ":" << port) - , GrpcClient(1) - , Service(GrpcClient.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(GrpcConf)) - , VanillaJobPath(vanillaJobPath) - , VanillaJobMd5(vanillaJobMd5) - , TaskScheduler(threads) - , RtTaskScheduler(1) - , OpenSessionTimeout(timeout) +namespace NYql { + +class TDqGateway: public IDqGateway +{ + enum ETaskPriority { + PRIO_NORMAL = 1, + PRIO_RT = 2 + }; + +public: + TDqGateway(const TString& host, int port, int threads, const TString& vanillaJobPath, const TString& vanillaJobMd5, TDuration timeout = TDuration::Minutes(60)) + : GrpcConf(TStringBuilder() << host << ":" << port) + , GrpcClient(1) + , Service(GrpcClient.CreateGRpcServiceConnection<Yql::DqsProto::DqService>(GrpcConf)) + , VanillaJobPath(vanillaJobPath) + , VanillaJobMd5(vanillaJobMd5) + , TaskScheduler(threads) + , RtTaskScheduler(1) + , OpenSessionTimeout(timeout) { TaskScheduler.Start(); - RtTaskScheduler.Start(); + RtTaskScheduler.Start(); } - - TString GetVanillaJobPath() override { - return VanillaJobPath; - } - - TString GetVanillaJobMd5() override { - return VanillaJobMd5; - } - - template<typename RespType> - void OnResponse(NThreading::TPromise<TResult> promise, TString sessionId, NGrpc::TGrpcStatus&& status, RespType&& resp, const THashMap<TString, TString>& modulesMapping, bool alwaysFallback = false) - { - YQL_LOG_CTX_SCOPE(sessionId); + + TString GetVanillaJobPath() override { + return VanillaJobPath; + } + + TString GetVanillaJobMd5() override { + return VanillaJobMd5; + } + + template<typename RespType> + void OnResponse(NThreading::TPromise<TResult> promise, TString sessionId, NGrpc::TGrpcStatus&& status, RespType&& resp, const THashMap<TString, TString>& modulesMapping, bool alwaysFallback = false) + { + YQL_LOG_CTX_SCOPE(sessionId); YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::callback"; { @@ -60,203 +60,203 @@ public: RunningQueries.erase(sessionId); } - TResult result; - - bool error = false; - bool fallback = false; - - if (status.Ok()) { + TResult result; + + bool error = false; + bool fallback = false; + + if (status.Ok()) { YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::Ok"; - - result.Truncated = resp.GetTruncated(); - - TOperationStatistics statistics; - - for (const auto& t : resp.GetMetric()) { + + result.Truncated = resp.GetTruncated(); + + TOperationStatistics statistics; + + for (const auto& t : resp.GetMetric()) { YQL_CLOG(TRACE, ProviderDq) << "Counter: " << t.GetName() << " : " << t.GetSum() << " : " << t.GetCount(); - TOperationStatistics::TEntry entry( - t.GetName(), - t.GetSum(), - t.GetMax(), - t.GetMin(), - t.GetAvg(), - t.GetCount()); - statistics.Entries.push_back(entry); - } - - result.Statistics = statistics; - - NYql::TIssues issues; - auto operation = resp.operation(); - for (auto& message : *operation.Mutableissues()) { - message.Setmessage(NBacktrace::Symbolize(message.Getmessage(), modulesMapping)); - } - NYql::IssuesFromMessage(operation.issues(), issues); - error = false; - for (const auto& issue : issues) { - if (issue.GetSeverity() <= TSeverityIds::S_ERROR) { - error = true; - } - if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR) { - fallback = true; - } - } - - // TODO: Save statistics in case of result failure - if (!error) { - Yql::DqsProto::ExecuteQueryResult queryResult; - resp.operation().result().UnpackTo(&queryResult); - result.Data = queryResult.yson().empty() + TOperationStatistics::TEntry entry( + t.GetName(), + t.GetSum(), + t.GetMax(), + t.GetMin(), + t.GetAvg(), + t.GetCount()); + statistics.Entries.push_back(entry); + } + + result.Statistics = statistics; + + NYql::TIssues issues; + auto operation = resp.operation(); + for (auto& message : *operation.Mutableissues()) { + message.Setmessage(NBacktrace::Symbolize(message.Getmessage(), modulesMapping)); + } + NYql::IssuesFromMessage(operation.issues(), issues); + error = false; + for (const auto& issue : issues) { + if (issue.GetSeverity() <= TSeverityIds::S_ERROR) { + error = true; + } + if (issue.GetCode() == TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR) { + fallback = true; + } + } + + // TODO: Save statistics in case of result failure + if (!error) { + Yql::DqsProto::ExecuteQueryResult queryResult; + resp.operation().result().UnpackTo(&queryResult); + result.Data = queryResult.yson().empty() ? NYdb::FormatResultSetYson(queryResult.result(), NYson::EYsonFormat::Binary) - : queryResult.yson(); - result.Issues.AddIssues(issues); - result.SetSuccess(); - } else { + : queryResult.yson(); + result.Issues.AddIssues(issues); + result.SetSuccess(); + } else { YQL_CLOG(ERROR, ProviderDq) << "Issue " << issues.ToString(); - result.Issues.AddIssues(issues); - if (fallback) { - result.Fallback = true; - result.SetSuccess(); - } - } - } else { + result.Issues.AddIssues(issues); + if (fallback) { + result.Fallback = true; + result.SetSuccess(); + } + } + } else { YQL_CLOG(ERROR, ProviderDq) << "Issue " << status.Msg; - auto issue = TIssue(TStringBuilder{} << "Error " << status.GRpcStatusCode << " message: " << status.Msg); - result.Retriable = status.GRpcStatusCode == grpc::CANCELLED; - if ((status.GRpcStatusCode == grpc::UNAVAILABLE /* terminating state */ - || status.GRpcStatusCode == grpc::CANCELLED /* server crashed or stopped before task process */) - || status.GRpcStatusCode == grpc::RESOURCE_EXHAUSTED /* send message limit */ - ) - { + auto issue = TIssue(TStringBuilder{} << "Error " << status.GRpcStatusCode << " message: " << status.Msg); + result.Retriable = status.GRpcStatusCode == grpc::CANCELLED; + if ((status.GRpcStatusCode == grpc::UNAVAILABLE /* terminating state */ + || status.GRpcStatusCode == grpc::CANCELLED /* server crashed or stopped before task process */) + || status.GRpcStatusCode == grpc::RESOURCE_EXHAUSTED /* send message limit */ + ) + { YQL_CLOG(ERROR, ProviderDq) << "Fallback " << status.GRpcStatusCode; - result.Fallback = true; - result.SetSuccess(); - result.Issues.AddIssue(issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR)); - } else { - error = true; - result.Issues.AddIssue(issue.SetCode(TIssuesIds::DQ_GATEWAY_ERROR, TSeverityIds::S_ERROR)); - } - } - - if (error && alwaysFallback) { + result.Fallback = true; + result.SetSuccess(); + result.Issues.AddIssue(issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR)); + } else { + error = true; + result.Issues.AddIssue(issue.SetCode(TIssuesIds::DQ_GATEWAY_ERROR, TSeverityIds::S_ERROR)); + } + } + + if (error && alwaysFallback) { YQL_CLOG(ERROR, ProviderDq) << "Force Fallback"; - result.Fallback = true; - result.ForceFallback = true; - result.SetSuccess(); - } - - if (!result.Success()) { - result.AddIssues(result.Issues); - } - - Async([promise=std::move(promise), result=std::move(result)]() mutable { promise.SetValue(result); }); - } - - NThreading::TFuture<void> Delay(TDuration duration, ETaskPriority prio = PRIO_NORMAL) { - NThreading::TPromise<void> promise = NThreading::NewPromise(); - - auto future = promise.GetFuture(); - - auto& taskScheduler = prio == PRIO_NORMAL ? TaskScheduler : RtTaskScheduler; - - if (!taskScheduler.Add(MakeIntrusive<TDelay>(promise), TInstant::Now() + duration)) { - promise.SetException("cannot delay"); - } - - return future; - } - - void Async(const std::function<void(void)>& f) { - NThreading::TPromise<void> promise = NThreading::NewPromise(); - - promise.GetFuture().Apply([=](const NThreading::TFuture<void>&) { - f(); - }); - - Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TDelay>(promise), TInstant())); - } - + result.Fallback = true; + result.ForceFallback = true; + result.SetSuccess(); + } + + if (!result.Success()) { + result.AddIssues(result.Issues); + } + + Async([promise=std::move(promise), result=std::move(result)]() mutable { promise.SetValue(result); }); + } + + NThreading::TFuture<void> Delay(TDuration duration, ETaskPriority prio = PRIO_NORMAL) { + NThreading::TPromise<void> promise = NThreading::NewPromise(); + + auto future = promise.GetFuture(); + + auto& taskScheduler = prio == PRIO_NORMAL ? TaskScheduler : RtTaskScheduler; + + if (!taskScheduler.Add(MakeIntrusive<TDelay>(promise), TInstant::Now() + duration)) { + promise.SetException("cannot delay"); + } + + return future; + } + + void Async(const std::function<void(void)>& f) { + NThreading::TPromise<void> promise = NThreading::NewPromise(); + + promise.GetFuture().Apply([=](const NThreading::TFuture<void>&) { + f(); + }); + + Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TDelay>(promise), TInstant())); + } + template <typename TResponse, typename TRequest, typename TStub> NThreading::TFuture<TResult> WithRetry( const TString& sessionId, const TRequest& queryPB, TStub stub, int retry, - const TDqSettings::TPtr& settings, - const THashMap<TString, TString>& modulesMapping + const TDqSettings::TPtr& settings, + const THashMap<TString, TString>& modulesMapping ) { - auto backoff = TDuration::MilliSeconds(settings->RetryBackoffMs.Get().GetOrElse(1000)); - auto promise = NThreading::NewPromise<TResult>(); - auto fallbackPolicy = settings->FallbackPolicy.Get().GetOrElse("default"); - auto alwaysFallback = fallbackPolicy == "always"; - auto callback = [this, promise, sessionId, alwaysFallback, modulesMapping](NGrpc::TGrpcStatus&& status, TResponse&& resp) mutable { - return OnResponse(std::move(promise), std::move(sessionId), std::move(status), std::move(resp), modulesMapping, alwaysFallback); - }; - + auto backoff = TDuration::MilliSeconds(settings->RetryBackoffMs.Get().GetOrElse(1000)); + auto promise = NThreading::NewPromise<TResult>(); + auto fallbackPolicy = settings->FallbackPolicy.Get().GetOrElse("default"); + auto alwaysFallback = fallbackPolicy == "always"; + auto callback = [this, promise, sessionId, alwaysFallback, modulesMapping](NGrpc::TGrpcStatus&& status, TResponse&& resp) mutable { + return OnResponse(std::move(promise), std::move(sessionId), std::move(status), std::move(resp), modulesMapping, alwaysFallback); + }; + Service->DoRequest<TRequest, TResponse>(queryPB, callback, stub); - - { - TGuard<TMutex> lock(ProgressMutex); - auto i = RunningQueries.find(sessionId); - if (i != RunningQueries.end()) { - if (i->second.first) { - ScheduleQueryStatusRequest(sessionId); - } - } else { - return NThreading::MakeFuture(TResult()); + + { + TGuard<TMutex> lock(ProgressMutex); + auto i = RunningQueries.find(sessionId); + if (i != RunningQueries.end()) { + if (i->second.first) { + ScheduleQueryStatusRequest(sessionId); + } + } else { + return NThreading::MakeFuture(TResult()); } } - return promise.GetFuture().Apply([=](const NThreading::TFuture<TResult>& result) { - if (result.HasException()) { - return result; - } - auto value = result.GetValue(); - if (value.Success() || retry == 0 || !value.Retriable) { - return result; - } - - return Delay(backoff) - .Apply([=](const NThreading::TFuture<void>& result) { - try { - result.TryRethrow(); - } catch (...) { - return NThreading::MakeErrorFuture<TResult>(std::current_exception()); - } - return WithRetry<TResponse>(sessionId, queryPB, stub, retry - 1, settings, modulesMapping); - }); - }); - } - + return promise.GetFuture().Apply([=](const NThreading::TFuture<TResult>& result) { + if (result.HasException()) { + return result; + } + auto value = result.GetValue(); + if (value.Success() || retry == 0 || !value.Retriable) { + return result; + } + + return Delay(backoff) + .Apply([=](const NThreading::TFuture<void>& result) { + try { + result.TryRethrow(); + } catch (...) { + return NThreading::MakeErrorFuture<TResult>(std::current_exception()); + } + return WithRetry<TResponse>(sessionId, queryPB, stub, retry - 1, settings, modulesMapping); + }); + }); + } + NThreading::TFuture<TResult> ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns, const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams, const TDqSettings::TPtr& settings, const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping, bool discard) override - { + { YQL_LOG_CTX_SCOPE(sessionId); - auto tasks = plan.GetTasks(); - - Yql::DqsProto::ExecuteGraphRequest queryPB; - for (const auto& task : tasks) { - auto* t = queryPB.AddTask(); - *t = task; - - Yql::DqsProto::TTaskMeta taskMeta; - task.GetMeta().UnpackTo(&taskMeta); - - for (auto& file : taskMeta.GetFiles()) { - YQL_ENSURE(!file.GetObjectId().empty()); - } - } - queryPB.SetSession(sessionId); - queryPB.SetResultType(plan.GetResultType()); - queryPB.SetSourceId(plan.GetSourceID().NodeId()-1); - for (const auto& column : columns) { - *queryPB.AddColumns() = column; - } - settings->Save(queryPB); + auto tasks = plan.GetTasks(); + + Yql::DqsProto::ExecuteGraphRequest queryPB; + for (const auto& task : tasks) { + auto* t = queryPB.AddTask(); + *t = task; + + Yql::DqsProto::TTaskMeta taskMeta; + task.GetMeta().UnpackTo(&taskMeta); + + for (auto& file : taskMeta.GetFiles()) { + YQL_ENSURE(!file.GetObjectId().empty()); + } + } + queryPB.SetSession(sessionId); + queryPB.SetResultType(plan.GetResultType()); + queryPB.SetSourceId(plan.GetSourceID().NodeId()-1); + for (const auto& column : columns) { + *queryPB.AddColumns() = column; + } + settings->Save(queryPB); { auto& secParams = *queryPB.MutableSecureParams(); @@ -264,7 +264,7 @@ public: secParams[k] = v; } } - + { auto& gParams = *queryPB.MutableGraphParams(); for (const auto&[k, v] : graphParams) { @@ -273,93 +273,93 @@ public: } queryPB.SetDiscard(discard); - - int retry = settings->MaxRetries.Get().GetOrElse(5); - - { - TGuard<TMutex> lock(ProgressMutex); - RunningQueries.emplace(sessionId, std::make_pair(progressWriter, TString(""))); - } - - YQL_LOG(DEBUG) << "Send query of size " << queryPB.ByteSizeLong(); - - return WithRetry<Yql::DqsProto::ExecuteGraphResponse>( - sessionId, - queryPB, - &Yql::DqsProto::DqService::Stub::AsyncExecuteGraph, - retry, + + int retry = settings->MaxRetries.Get().GetOrElse(5); + + { + TGuard<TMutex> lock(ProgressMutex); + RunningQueries.emplace(sessionId, std::make_pair(progressWriter, TString(""))); + } + + YQL_LOG(DEBUG) << "Send query of size " << queryPB.ByteSizeLong(); + + return WithRetry<Yql::DqsProto::ExecuteGraphResponse>( + sessionId, + queryPB, + &Yql::DqsProto::DqService::Stub::AsyncExecuteGraph, + retry, settings, - modulesMapping); - } - + modulesMapping); + } + NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override { YQL_LOG_CTX_SCOPE(sessionId); YQL_CLOG(INFO, ProviderDq) << "OpenSession"; - Yql::DqsProto::OpenSessionRequest request; - request.SetSession(sessionId); + Yql::DqsProto::OpenSessionRequest request; + request.SetSession(sessionId); request.SetUsername(username); - - NGrpc::TCallMeta meta; - meta.Timeout = OpenSessionTimeout; - - auto promise = NThreading::NewPromise<void>(); - auto callback = [this, promise, sessionId](NGrpc::TGrpcStatus&& status, Yql::DqsProto::OpenSessionResponse&& resp) mutable { - Y_UNUSED(resp); + + NGrpc::TCallMeta meta; + meta.Timeout = OpenSessionTimeout; + + auto promise = NThreading::NewPromise<void>(); + auto callback = [this, promise, sessionId](NGrpc::TGrpcStatus&& status, Yql::DqsProto::OpenSessionResponse&& resp) mutable { + Y_UNUSED(resp); YQL_LOG_CTX_SCOPE(sessionId); - if (status.Ok()) { + if (status.Ok()) { YQL_CLOG(INFO, ProviderDq) << "OpenSession OK"; - SchedulePingSessionRequest(sessionId); - Async([promise=std::move(promise)]() mutable { promise.SetValue(); }); - } else { + SchedulePingSessionRequest(sessionId); + Async([promise=std::move(promise)]() mutable { promise.SetValue(); }); + } else { YQL_CLOG(ERROR, ProviderDq) << "OpenSession error: " << status.Msg; - Async([promise=std::move(promise), status]() mutable { promise.SetException(status.Msg); }); - } - }; - - Service->DoRequest<Yql::DqsProto::OpenSessionRequest, Yql::DqsProto::OpenSessionResponse>( - request, callback, &Yql::DqsProto::DqService::Stub::AsyncOpenSession, meta); - return promise.GetFuture(); - } - - void CloseSession(const TString& sessionId) override { - Yql::DqsProto::CloseSessionRequest request; - request.SetSession(sessionId); - - auto callback = [](NGrpc::TGrpcStatus&& status, Yql::DqsProto::CloseSessionResponse&& resp) { - Y_UNUSED(resp); - Y_UNUSED(status); - }; - - { - TGuard<TMutex> lock(ProgressMutex); - RunningQueries.erase(sessionId); - } - - Service->DoRequest<Yql::DqsProto::CloseSessionRequest, Yql::DqsProto::CloseSessionResponse>( - request, callback, &Yql::DqsProto::DqService::Stub::AsyncCloseSession); - } - + Async([promise=std::move(promise), status]() mutable { promise.SetException(status.Msg); }); + } + }; + + Service->DoRequest<Yql::DqsProto::OpenSessionRequest, Yql::DqsProto::OpenSessionResponse>( + request, callback, &Yql::DqsProto::DqService::Stub::AsyncOpenSession, meta); + return promise.GetFuture(); + } + + void CloseSession(const TString& sessionId) override { + Yql::DqsProto::CloseSessionRequest request; + request.SetSession(sessionId); + + auto callback = [](NGrpc::TGrpcStatus&& status, Yql::DqsProto::CloseSessionResponse&& resp) { + Y_UNUSED(resp); + Y_UNUSED(status); + }; + + { + TGuard<TMutex> lock(ProgressMutex); + RunningQueries.erase(sessionId); + } + + Service->DoRequest<Yql::DqsProto::CloseSessionRequest, Yql::DqsProto::CloseSessionResponse>( + request, callback, &Yql::DqsProto::DqService::Stub::AsyncCloseSession); + } + void RequestQueryStatus(const TString& sessionId) { Yql::DqsProto::QueryStatusRequest request; request.SetSession(sessionId); - IDqGateway::TPtr self = this; - auto callback = [this, self, sessionId](NGrpc::TGrpcStatus&& status, Yql::DqsProto::QueryStatusResponse&& resp) { + IDqGateway::TPtr self = this; + auto callback = [this, self, sessionId](NGrpc::TGrpcStatus&& status, Yql::DqsProto::QueryStatusResponse&& resp) { if (status.Ok()) { - TGuard<TMutex> lock(ProgressMutex); + TGuard<TMutex> lock(ProgressMutex); TString stage; TDqProgressWriter* dqProgressWriter = nullptr; - auto it = RunningQueries.find(sessionId); - if (it != RunningQueries.end()) { - dqProgressWriter = &it->second.first; - auto lastStatus = it->second.second; - if (dqProgressWriter && lastStatus != resp.GetStatus()) { - stage = resp.GetStatus(); - it->second.second = stage; - } - - ScheduleQueryStatusRequest(sessionId); + auto it = RunningQueries.find(sessionId); + if (it != RunningQueries.end()) { + dqProgressWriter = &it->second.first; + auto lastStatus = it->second.second; + if (dqProgressWriter && lastStatus != resp.GetStatus()) { + stage = resp.GetStatus(); + it->second.second = stage; + } + + ScheduleQueryStatusRequest(sessionId); } - + if (!stage.empty() && dqProgressWriter) { (*dqProgressWriter)(stage); } @@ -374,7 +374,7 @@ public: } void ScheduleQueryStatusRequest(const TString& sessionId) { - Delay(TDuration::MilliSeconds(1000)).Subscribe([this, sessionId](NThreading::TFuture<void> fut) { + Delay(TDuration::MilliSeconds(1000)).Subscribe([this, sessionId](NThreading::TFuture<void> fut) { if (fut.HasException()) { TGuard<TMutex> lock(ProgressMutex); RunningQueries.erase(sessionId); @@ -388,27 +388,27 @@ public: }); } - void SchedulePingSessionRequest(const TString& sessionId) { - auto callback = [this, sessionId]( - NGrpc::TGrpcStatus&& status, - Yql::DqsProto::PingSessionResponse&&) mutable - { - if (status.GRpcStatusCode == grpc::INVALID_ARGUMENT) { - YQL_LOG(INFO) << "Session closed " << sessionId; - } else { - SchedulePingSessionRequest(sessionId); - } - }; - Delay(TDuration::Seconds(10), PRIO_RT).Subscribe([this, callback, sessionId](const NThreading::TFuture<void>&) { - Yql::DqsProto::PingSessionRequest query; - query.SetSession(sessionId); - Service->DoRequest<Yql::DqsProto::PingSessionRequest, Yql::DqsProto::PingSessionResponse>( - query, - callback, - &Yql::DqsProto::DqService::Stub::AsyncPingSession); - }); - } - + void SchedulePingSessionRequest(const TString& sessionId) { + auto callback = [this, sessionId]( + NGrpc::TGrpcStatus&& status, + Yql::DqsProto::PingSessionResponse&&) mutable + { + if (status.GRpcStatusCode == grpc::INVALID_ARGUMENT) { + YQL_LOG(INFO) << "Session closed " << sessionId; + } else { + SchedulePingSessionRequest(sessionId); + } + }; + Delay(TDuration::Seconds(10), PRIO_RT).Subscribe([this, callback, sessionId](const NThreading::TFuture<void>&) { + Yql::DqsProto::PingSessionRequest query; + query.SetSession(sessionId); + Service->DoRequest<Yql::DqsProto::PingSessionRequest, Yql::DqsProto::PingSessionResponse>( + query, + callback, + &Yql::DqsProto::DqService::Stub::AsyncPingSession); + }); + } + struct TDelay: public TTaskScheduler::ITask { TDelay(NThreading::TPromise<void> p) : Promise(std::move(p)) @@ -422,32 +422,32 @@ public: NThreading::TPromise<void> Promise; }; -private: - NGrpc::TGRpcClientConfig GrpcConf; - NGrpc::TGRpcClientLow GrpcClient; - std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> Service; - - TMutex ProgressMutex; - TMutex Mutex; +private: + NGrpc::TGRpcClientConfig GrpcConf; + NGrpc::TGRpcClientLow GrpcClient; + std::unique_ptr<NGrpc::TServiceConnection<Yql::DqsProto::DqService>> Service; + + TMutex ProgressMutex; + TMutex Mutex; THashMap<TString, std::pair<TDqProgressWriter, TString>> RunningQueries; - TString VanillaJobPath; - TString VanillaJobMd5; - - TTaskScheduler TaskScheduler; - TTaskScheduler RtTaskScheduler; - - const TDuration OpenSessionTimeout; -}; - -TIntrusivePtr<IDqGateway> CreateDqGateway(const TString& host, int port, int threads) { - return new TDqGateway(host, port, threads, "", ""); -} - -TIntrusivePtr<IDqGateway> CreateDqGateway(const NProto::TDqConfig& config) { - return new TDqGateway("localhost", config.GetPort(), 8, - config.GetYtBackends()[0].GetVanillaJob(), - config.GetYtBackends()[0].GetVanillaJobMd5(), - TDuration::Seconds(15)); -} - -} // namespace NYql + TString VanillaJobPath; + TString VanillaJobMd5; + + TTaskScheduler TaskScheduler; + TTaskScheduler RtTaskScheduler; + + const TDuration OpenSessionTimeout; +}; + +TIntrusivePtr<IDqGateway> CreateDqGateway(const TString& host, int port, int threads) { + return new TDqGateway(host, port, threads, "", ""); +} + +TIntrusivePtr<IDqGateway> CreateDqGateway(const NProto::TDqConfig& config) { + return new TDqGateway("localhost", config.GetPort(), 8, + config.GetYtBackends()[0].GetVanillaJob(), + config.GetYtBackends()[0].GetVanillaJobMd5(), + TDuration::Seconds(15)); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h index 3400c0d3fb..d562776033 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h @@ -1,68 +1,68 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/ast/yql_expr.h> - + #include <ydb/library/yql/providers/common/gateway/yql_provider_gateway.h> #include <ydb/library/yql/providers/dq/api/protos/service.pb.h> #include <ydb/library/yql/providers/dq/planner/execution_planner.h> #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h> #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h> - + #include <ydb/library/yql/core/yql_udf_resolver.h> #include <ydb/library/yql/core/yql_execution.h> - + #include <ydb/library/yql/minikql/mkql_function_registry.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> - -namespace NYql { - + +namespace NYql { + namespace NProto { class TDqConfig; } -class IDqGateway : public TThrRefBase { -public: - using TPtr = TIntrusivePtr<IDqGateway>; - using TFileResource = Yql::DqsProto::TFile; +class IDqGateway : public TThrRefBase { +public: + using TPtr = TIntrusivePtr<IDqGateway>; + using TFileResource = Yql::DqsProto::TFile; using TDqProgressWriter = std::function<void(const TString&)>; - - struct TFileResourceHash { - std::size_t operator()(const TFileResource& f) const { - return std::hash<TString>()(f.GetObjectId()); - } - }; - - struct TFileResourceEqual { - bool operator()(const TFileResource& a, const TFileResource& b) const { - return a.GetObjectId() == b.GetObjectId(); - } - }; - - using TUploadList = THashSet<TFileResource, TFileResourceHash, TFileResourceEqual>; - + + struct TFileResourceHash { + std::size_t operator()(const TFileResource& f) const { + return std::hash<TString>()(f.GetObjectId()); + } + }; + + struct TFileResourceEqual { + bool operator()(const TFileResource& a, const TFileResource& b) const { + return a.GetObjectId() == b.GetObjectId(); + } + }; + + using TUploadList = THashSet<TFileResource, TFileResourceHash, TFileResourceEqual>; + class TResult: public NCommon::TOperationResult { - public: - TString Data; - bool Fallback = false; - bool ForceFallback = false; - bool Retriable = false; - bool Truncated = false; + public: + TString Data; + bool Fallback = false; + bool ForceFallback = false; + bool Retriable = false; + bool Truncated = false; ui64 RowsCount = 0; - TIssues Issues; - - TOperationStatistics Statistics; - - TResult() = default; - }; - - virtual ~IDqGateway() = default; - + TIssues Issues; + + TOperationStatistics Statistics; + + TResult() = default; + }; + + virtual ~IDqGateway() = default; + virtual NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) = 0; - - virtual void CloseSession(const TString& sessionId) = 0; - + + virtual void CloseSession(const TString& sessionId) = 0; + virtual NThreading::TFuture<TResult> ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns, const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams, @@ -70,16 +70,16 @@ public: const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping, bool discard) = 0; - virtual TString GetVanillaJobPath() { - return ""; - } - - virtual TString GetVanillaJobMd5() { - return ""; - } -}; - -TIntrusivePtr<IDqGateway> CreateDqGateway(const TString& host, int port, int threads); -TIntrusivePtr<IDqGateway> CreateDqGateway(const NProto::TDqConfig& config); - -} // namespace NYql + virtual TString GetVanillaJobPath() { + return ""; + } + + virtual TString GetVanillaJobMd5() { + return ""; + } +}; + +TIntrusivePtr<IDqGateway> CreateDqGateway(const TString& host, int port, int threads); +TIntrusivePtr<IDqGateway> CreateDqGateway(const NProto::TDqConfig& config); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp index 6ea4e85572..69788c5c76 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp @@ -1,111 +1,111 @@ -#include "yql_dq_provider.h" -#include "yql_dq_state.h" - -#include "yql_dq_datasink.h" -#include "yql_dq_datasource.h" - +#include "yql_dq_provider.h" +#include "yql_dq_state.h" + +#include "yql_dq_datasink.h" +#include "yql_dq_datasource.h" + #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> - + #include <ydb/library/yql/utils/log/log.h> - -namespace NYql { - + +namespace NYql { + TDataProviderInitializer GetDqDataProviderInitializer( TExecTransformerFactory execTransformerFactory, - const IDqGateway::TPtr& dqGateway, + const IDqGateway::TPtr& dqGateway, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, - const IMetricsRegistryPtr& metrics, - const TFileStoragePtr& fileStorage, - bool externalUser) -{ + const IMetricsRegistryPtr& metrics, + const TFileStoragePtr& fileStorage, + bool externalUser) +{ return [execTransformerFactory, dqGateway, compFactory, metrics, fileStorage, externalUser] ( - const TString& userName, - const TString& sessionId, - const TGatewaysConfig* gatewaysConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, - TIntrusivePtr<IRandomProvider> randomProvider, - TIntrusivePtr<TTypeAnnotationContext> typeCtx, - const TOperationProgressWriter& progressWriter, - const TYqlOperationOptions& operationOptions - ) { - Y_UNUSED(userName); - + const TString& userName, + const TString& sessionId, + const TGatewaysConfig* gatewaysConfig, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<IRandomProvider> randomProvider, + TIntrusivePtr<TTypeAnnotationContext> typeCtx, + const TOperationProgressWriter& progressWriter, + const TYqlOperationOptions& operationOptions + ) { + Y_UNUSED(userName); + TDqStatePtr state = MakeIntrusive<TDqState>( - dqGateway, // nullptr for yqlrun - gatewaysConfig, - functionRegistry, + dqGateway, // nullptr for yqlrun + gatewaysConfig, + functionRegistry, compFactory, - randomProvider, - typeCtx.Get(), - progressWriter, - operationOptions, - sessionId, - metrics, // nullptr for yqlrun - fileStorage, - dqGateway ? dqGateway->GetVanillaJobPath() : "", - dqGateway ? dqGateway->GetVanillaJobMd5() : "", - externalUser - ); - - TDataProviderInfo info; + randomProvider, + typeCtx.Get(), + progressWriter, + operationOptions, + sessionId, + metrics, // nullptr for yqlrun + fileStorage, + dqGateway ? dqGateway->GetVanillaJobPath() : "", + dqGateway ? dqGateway->GetVanillaJobMd5() : "", + externalUser + ); + + TDataProviderInfo info; info.Names.insert(TString{DqProviderName}); - + info.Source = CreateDqDataSource(state, execTransformerFactory); info.Sink = CreateDqDataSink(state); info.OpenSession = [dqGateway, metrics, gatewaysConfig, state]( - const TString& sessionId, - const TString& username, - const TOperationProgressWriter& progressWriter, - const TYqlOperationOptions& operationOptions, - TIntrusivePtr<IRandomProvider> randomProvider, - TIntrusivePtr<ITimeProvider> timeProvider) { - - if (metrics) { - metrics->IncCounter("dq", "OpenSession"); - } - - if (gatewaysConfig) { - state->Settings->Init(gatewaysConfig->GetDq(), username); - } - - Y_UNUSED(progressWriter); - Y_UNUSED(operationOptions); - Y_UNUSED(randomProvider); - Y_UNUSED(timeProvider); - if (dqGateway) { // nullptr in yqlrun - auto t = TInstant::Now(); - YQL_LOG(DEBUG) << "OpenSession " << sessionId; + const TString& sessionId, + const TString& username, + const TOperationProgressWriter& progressWriter, + const TYqlOperationOptions& operationOptions, + TIntrusivePtr<IRandomProvider> randomProvider, + TIntrusivePtr<ITimeProvider> timeProvider) { + + if (metrics) { + metrics->IncCounter("dq", "OpenSession"); + } + + if (gatewaysConfig) { + state->Settings->Init(gatewaysConfig->GetDq(), username); + } + + Y_UNUSED(progressWriter); + Y_UNUSED(operationOptions); + Y_UNUSED(randomProvider); + Y_UNUSED(timeProvider); + if (dqGateway) { // nullptr in yqlrun + auto t = TInstant::Now(); + YQL_LOG(DEBUG) << "OpenSession " << sessionId; auto future = dqGateway->OpenSession(sessionId, username); - future.Subscribe([sessionId, t] (const auto& ) { - YQL_LOG(DEBUG) << "OpenSession " << sessionId << " complete in " << (TInstant::Now()-t).MilliSeconds(); - }); - return future; - } else { - return NThreading::MakeFuture(); - } - }; - - info.CloseSession = [dqGateway, metrics](const TString& sessionId) { - if (metrics) { - metrics->IncCounter("dq", "CloseSession"); - } - - if (dqGateway) { // nullptr in yqlrun - YQL_LOG(DEBUG) << "CloseSession " << sessionId; - dqGateway->CloseSession(sessionId); - } - }; - - info.TokenResolver = [](const TString& url) -> TMaybe<TString> { - Y_UNUSED(url); - - return Nothing(); - }; - - return info; - }; -} - -} // namespace NYql + future.Subscribe([sessionId, t] (const auto& ) { + YQL_LOG(DEBUG) << "OpenSession " << sessionId << " complete in " << (TInstant::Now()-t).MilliSeconds(); + }); + return future; + } else { + return NThreading::MakeFuture(); + } + }; + + info.CloseSession = [dqGateway, metrics](const TString& sessionId) { + if (metrics) { + metrics->IncCounter("dq", "CloseSession"); + } + + if (dqGateway) { // nullptr in yqlrun + YQL_LOG(DEBUG) << "CloseSession " << sessionId; + dqGateway->CloseSession(sessionId); + } + }; + + info.TokenResolver = [](const TString& url) -> TMaybe<TString> { + Y_UNUSED(url); + + return Nothing(); + }; + + return info; + }; +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_provider.h b/ydb/library/yql/providers/dq/provider/yql_dq_provider.h index 404b3404ba..6e4aff4ee9 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_provider.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_provider.h @@ -1,15 +1,15 @@ -#pragma once - -#include "yql_dq_gateway.h" - +#pragma once + +#include "yql_dq_gateway.h" + #include <ydb/library/yql/providers/common/metrics/metrics_registry.h> #include <ydb/library/yql/core/yql_data_provider.h> #include <ydb/library/yql/core/yql_udf_resolver.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> #include <ydb/library/yql/core/file_storage/file_storage.h> - -namespace NYql { - + +namespace NYql { + struct TDqState; using TDqStatePtr = TIntrusivePtr<TDqState>; @@ -17,10 +17,10 @@ using TExecTransformerFactory = std::function<IGraphTransformer*(const TDqStateP TDataProviderInitializer GetDqDataProviderInitializer( TExecTransformerFactory execTransformerFactory, - const IDqGateway::TPtr& dqGateway, + const IDqGateway::TPtr& dqGateway, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, - const IMetricsRegistryPtr& metrics, - const TFileStoragePtr& fileStorage, - bool externalUser = false); - -} // namespace NYql + const IMetricsRegistryPtr& metrics, + const TFileStoragePtr& fileStorage, + bool externalUser = false); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp index ed1def2744..90e8af759c 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp +++ b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp @@ -13,8 +13,8 @@ #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/dq/opt/dq_opt.h> -#include <util/generic/scope.h> - +#include <util/generic/scope.h> + namespace NYql { using namespace NNodes; @@ -27,11 +27,11 @@ const THashSet<TStringBuf> UNSUPPORTED_CALLABLE = { TCoForwardList::CallableName } -namespace NDq { - bool CheckJoinColumns(const TExprBase& node); - bool CheckJoinLinkSettings(const TExprBase& node); -} // namespace NDq - +namespace NDq { + bool CheckJoinColumns(const TExprBase& node); + bool CheckJoinLinkSettings(const TExprBase& node); +} // namespace NDq + class TDqsRecaptureTransformer : public TSyncTransformerBase { public: TDqsRecaptureTransformer(TDqStatePtr state) @@ -45,58 +45,58 @@ public: return TStatus::Ok; } - Y_SCOPE_EXIT(&) { - FlushStatistics(); - }; - - if (State_->ExternalUser) { - Statistics_["DqExternalUser"]++; - return TStatus::Ok; - } - - if (State_->TypeCtx->ForceDq) { - Statistics_["DqForce"]++; - } - + Y_SCOPE_EXIT(&) { + FlushStatistics(); + }; + + if (State_->ExternalUser) { + Statistics_["DqExternalUser"]++; + return TStatus::Ok; + } + + if (State_->TypeCtx->ForceDq) { + Statistics_["DqForce"]++; + } + if (!State_->TypeCtx->ForceDq) { - if (!State_->Settings->AnalyzeQuery.Get().GetOrElse(false)) { - Statistics_["DqAnalyzerOff"]++; - } - - if (State_->TypeCtx->PureResultDataSource != DqProviderName) { - Statistics_["DqPureResultDataSourceMismatch"]++; - } - + if (!State_->Settings->AnalyzeQuery.Get().GetOrElse(false)) { + Statistics_["DqAnalyzerOff"]++; + } + + if (State_->TypeCtx->PureResultDataSource != DqProviderName) { + Statistics_["DqPureResultDataSourceMismatch"]++; + } + if (State_->TypeCtx->PureResultDataSource != DqProviderName || !State_->Settings->AnalyzeQuery.Get().GetOrElse(false)) { return TStatus::Ok; } - Statistics_["DqAnalyzerOn"]++; - + Statistics_["DqAnalyzerOn"]++; + ui64 dataSize = 0; bool good = true; - bool hasJoin = false; + bool hasJoin = false; TNodeSet visited; - Scan(*input, ctx, good, dataSize, visited, hasJoin); - - if (good) { - Statistics_["DqAnalyzerOk"]++; - } else { - Statistics_["DqAnalyzerFail"] ++; - } - - if ((hasJoin && dataSize > State_->Settings->MaxDataSizePerQuery.Get().GetOrElse(10_GB))) { - Statistics_["DqAnalyzerBigJoin"]++; - } - - if (!good || (hasJoin && dataSize > State_->Settings->MaxDataSizePerQuery.Get().GetOrElse(10_GB))) { - YQL_LOG(DEBUG) << "good: " << good << " hasJoin: " << hasJoin << " dataSize: " << dataSize; + Scan(*input, ctx, good, dataSize, visited, hasJoin); + + if (good) { + Statistics_["DqAnalyzerOk"]++; + } else { + Statistics_["DqAnalyzerFail"] ++; + } + + if ((hasJoin && dataSize > State_->Settings->MaxDataSizePerQuery.Get().GetOrElse(10_GB))) { + Statistics_["DqAnalyzerBigJoin"]++; + } + + if (!good || (hasJoin && dataSize > State_->Settings->MaxDataSizePerQuery.Get().GetOrElse(10_GB))) { + YQL_LOG(DEBUG) << "good: " << good << " hasJoin: " << hasJoin << " dataSize: " << dataSize; return TStatus::Ok; } } - State_->TypeCtx->DqFallbackPolicy = State_->Settings->FallbackPolicy.Get().GetOrElse("default"); - + State_->TypeCtx->DqFallbackPolicy = State_->Settings->FallbackPolicy.Get().GetOrElse("default"); + auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) { if (auto maybeRead = TMaybeNode<TCoRight>(node).Input()) { if (maybeRead.Raw()->ChildrenSize() > 1 && TCoDataSource::Match(maybeRead.Raw()->Child(1))) { @@ -116,8 +116,8 @@ public: }, ctx, TOptimizeExprSettings{State_->TypeCtx}); if (input != output) { - // TODO: Add before/after recapture transformers - State_->TypeCtx->DqCaptured = true; + // TODO: Add before/after recapture transformers + State_->TypeCtx->DqCaptured = true; // TODO: drop this after implementing DQS ConstraintTransformer State_->TypeCtx->ExpectedConstraints.clear(); } @@ -128,27 +128,27 @@ public: } private: - void AddInfo(TExprContext& ctx, const TString& message) const { - YQL_LOG(DEBUG) << message; - TIssue info("DQ cannot execute the query. Cause: " + message); - info.Severity = TSeverityIds::S_INFO; + void AddInfo(TExprContext& ctx, const TString& message) const { + YQL_LOG(DEBUG) << message; + TIssue info("DQ cannot execute the query. Cause: " + message); + info.Severity = TSeverityIds::S_INFO; ctx.IssueManager.RaiseIssue(info); - } - + } + void Scan(const TExprNode& node, TExprContext& ctx, bool& good, ui64& dataSize, TNodeSet& visited, bool& hasJoin) const { if (!visited.insert(&node).second) { return; } - TExprBase expr(&node); - if (TMaybeNode<TCoEquiJoin>(&node)) { - hasJoin = true; - } - + TExprBase expr(&node); + if (TMaybeNode<TCoEquiJoin>(&node)) { + hasJoin = true; + } + if (TCoCommit::Match(&node)) { for (size_t i = 0; i != node.ChildrenSize() && good; ++i) { if (i != TCoCommit::idx_DataSink) { - Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); + Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); } } } else if (node.IsCallable(UNSUPPORTED_CALLABLE)) { @@ -169,12 +169,12 @@ private: AddInfo(ctx, TStringBuilder() << "sink '" << datasink.Cast().Value() << "' is not supported by DQ"); good = false; } - } else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinColumns(expr)) { - AddInfo(ctx, TStringBuilder() << "unsupported join column"); - good = false; - } else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinLinkSettings(expr)) { - AddInfo(ctx, TStringBuilder() << "unsupported join any"); - good = false; + } else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinColumns(expr)) { + AddInfo(ctx, TStringBuilder() << "unsupported join column"); + good = false; + } else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinLinkSettings(expr)) { + AddInfo(ctx, TStringBuilder() << "unsupported join any"); + good = false; } else if (node.ChildrenSize() > 1 && TCoDataSource::Match(node.Child(1))) { auto dataSourceName = node.Child(1)->Child(0)->Content(); if (dataSourceName != DqProviderName && !node.IsCallable(ConfigureName)) { @@ -193,7 +193,7 @@ private: } if (good) { - Scan(node.Head(), ctx,good, dataSize, visited, hasJoin); + Scan(node.Head(), ctx,good, dataSize, visited, hasJoin); } } else if (node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::World && !TCoCommit::Match(&node) @@ -214,7 +214,7 @@ private: } if (good) { for (size_t i = 0; i != node.ChildrenSize() && good; ++i) { - Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); + Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); } } } @@ -231,29 +231,29 @@ private: } else { for (size_t i = 0; i != node.ChildrenSize() && good; ++i) { - Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); + Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin); } } } private: TDqStatePtr State_; - - THashMap<TString, int> Statistics_; - - void FlushStatistics() { - TOperationStatistics statistics; - for (const auto& [k, v] : Statistics_) { - if (v == 1) { - statistics.Entries.push_back(TOperationStatistics::TEntry(k, 0, 0, 0, 0, 1)); - } - } - - TGuard<TMutex> lock(State_->Mutex); - if (!statistics.Entries.empty()) { - State_->Statistics[State_->MetricId++] = statistics; - } - } + + THashMap<TString, int> Statistics_; + + void FlushStatistics() { + TOperationStatistics statistics; + for (const auto& [k, v] : Statistics_) { + if (v == 1) { + statistics.Entries.push_back(TOperationStatistics::TEntry(k, 0, 0, 0, 0, 1)); + } + } + + TGuard<TMutex> lock(State_->Mutex); + if (!statistics.Entries.empty()) { + State_->Statistics[State_->MetricId++] = statistics; + } + } }; THolder<IGraphTransformer> CreateDqsRecaptureTransformer(TDqStatePtr state) { diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_state.h b/ydb/library/yql/providers/dq/provider/yql_dq_state.h index 38dd29d30b..a3e54dcb11 100644 --- a/ydb/library/yql/providers/dq/provider/yql_dq_state.h +++ b/ydb/library/yql/providers/dq/provider/yql_dq_state.h @@ -1,72 +1,72 @@ -#pragma once - -#include "yql_dq_gateway.h" - +#pragma once + +#include "yql_dq_gateway.h" + #include <ydb/library/yql/providers/common/metrics/metrics_registry.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> - + #include <util/generic/ptr.h> -namespace NYql { - -using namespace NDqs; // TODO: remove this namespace; - +namespace NYql { + +using namespace NDqs; // TODO: remove this namespace; + struct TDqState: public TThrRefBase { - IDqGateway::TPtr DqGateway; - const TGatewaysConfig* GatewaysConfig; - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + IDqGateway::TPtr DqGateway; + const TGatewaysConfig* GatewaysConfig; + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory; - TIntrusivePtr<IRandomProvider> RandomProvider; - TTypeAnnotationContext* TypeCtx; - const TOperationProgressWriter ProgressWriter; - const TYqlOperationOptions OperationOptions; - const TString SessionId; - const IMetricsRegistryPtr Metrics; - const TFileStoragePtr FileStorage; - const TString VanillaJobPath; - const TString VanillaJobMd5; - TString YtToken; - TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); - bool ExternalUser; - - TMutex Mutex; - THashMap<ui32, TOperationStatistics> Statistics; - std::atomic<ui32> MetricId = 1; - - TDqState( - const IDqGateway::TPtr& dqGateway, - const TGatewaysConfig* gatewaysConfig, - const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, + TIntrusivePtr<IRandomProvider> RandomProvider; + TTypeAnnotationContext* TypeCtx; + const TOperationProgressWriter ProgressWriter; + const TYqlOperationOptions OperationOptions; + const TString SessionId; + const IMetricsRegistryPtr Metrics; + const TFileStoragePtr FileStorage; + const TString VanillaJobPath; + const TString VanillaJobMd5; + TString YtToken; + TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); + bool ExternalUser; + + TMutex Mutex; + THashMap<ui32, TOperationStatistics> Statistics; + std::atomic<ui32> MetricId = 1; + + TDqState( + const IDqGateway::TPtr& dqGateway, + const TGatewaysConfig* gatewaysConfig, + const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, - const TIntrusivePtr<IRandomProvider>& randomProvider, - TTypeAnnotationContext* typeCtx, - const TOperationProgressWriter& progressWriter, - const TYqlOperationOptions& operationOptions, - const TString& sessionId, - const IMetricsRegistryPtr& metrics, - const TFileStoragePtr& fileStorage, - const TString& vanillaJobPath, - const TString& vanillaJobMd5, - bool externalUser) - : DqGateway(dqGateway) - , GatewaysConfig(gatewaysConfig) - , FunctionRegistry(functionRegistry) + const TIntrusivePtr<IRandomProvider>& randomProvider, + TTypeAnnotationContext* typeCtx, + const TOperationProgressWriter& progressWriter, + const TYqlOperationOptions& operationOptions, + const TString& sessionId, + const IMetricsRegistryPtr& metrics, + const TFileStoragePtr& fileStorage, + const TString& vanillaJobPath, + const TString& vanillaJobMd5, + bool externalUser) + : DqGateway(dqGateway) + , GatewaysConfig(gatewaysConfig) + , FunctionRegistry(functionRegistry) , ComputationFactory(compFactory) - , RandomProvider(randomProvider) - , TypeCtx(typeCtx) - , ProgressWriter(progressWriter) - , OperationOptions(operationOptions) - , SessionId(sessionId) - , Metrics(metrics) - , FileStorage(fileStorage) - , VanillaJobPath(vanillaJobPath) - , VanillaJobMd5(vanillaJobMd5) - , ExternalUser(externalUser) - { } -}; - + , RandomProvider(randomProvider) + , TypeCtx(typeCtx) + , ProgressWriter(progressWriter) + , OperationOptions(operationOptions) + , SessionId(sessionId) + , Metrics(metrics) + , FileStorage(fileStorage) + , VanillaJobPath(vanillaJobPath) + , VanillaJobMd5(vanillaJobMd5) + , ExternalUser(externalUser) + { } +}; + using TDqStatePtr = TIntrusivePtr<TDqState>; - -} // namespace + +} // namespace diff --git a/ydb/library/yql/providers/dq/runtime/file_cache.cpp b/ydb/library/yql/providers/dq/runtime/file_cache.cpp index b4d37e0fac..18005eb97f 100644 --- a/ydb/library/yql/providers/dq/runtime/file_cache.cpp +++ b/ydb/library/yql/providers/dq/runtime/file_cache.cpp @@ -1,165 +1,165 @@ -#include "file_cache.h" - +#include "file_cache.h" + #include <ydb/library/yql/utils/log/log.h> - -#include <util/system/fs.h> -#include <util/system/file.h> -#include <util/folder/path.h> -#include <util/folder/iterator.h> - -namespace NYql { - -TFileCache::TFileCache(const TString& baseDir, i64 size) - : BaseDir(baseDir) - , TotalSize(size) -{ - NFs::MakeDirectoryRecursive(BaseDir, NFs::FP_NONSECRET_FILE, false); - - BaseDir = TFsPath(BaseDir).RealPath().GetPath(); - - Scan(); -} - -void TFileCache::Scan() -{ - TDirIterator it( - BaseDir, - TDirIterator::TOptions() - .SetMaxLevel(2)); - - TVector<TFileObject> allFiles; - for (const auto& item : it) { - if (item.fts_level != 2) { - continue; - } - TFsPath path = item.fts_path; - if (!path.IsFile()) { - continue; - } - auto split = path.PathSplit(); - - TFileObject file; - file.ObjectId = TString(split[split.size() - 2]) + TString(split[split.size() - 1]); - file.Name = TString(split[split.size() - 1]); -#ifdef _linux_ - file.LastAccess = item.fts_statp->st_atim.tv_sec; -#endif - file.Size = item.fts_statp->st_size; - - UsedSize += file.Size; - - YQL_LOG(DEBUG) << file.Name << "|" << file.ObjectId; - - allFiles.emplace_back(std::move(file)); - } - - std::sort(allFiles.begin(), allFiles.end(), [](const auto&a, const auto& b) { - return a.LastAccess < b.LastAccess; - }); - - TGuard<TMutex> guard(Mutex); - for (auto& file : allFiles) { - LRU.push_back(file.ObjectId); - file.Position = --LRU.end(); - Files.insert(std::make_pair(file.ObjectId, file)); - } - - Clean(); -} - -ui64 TFileCache::FreeDiskSize() { - return TotalSize - UsedSize; -} - -ui64 TFileCache::UsedDiskSize() { - return UsedSize; -} - -void TFileCache::Clean() { - while (UsedSize > TotalSize && LRU.size() > 2) { - auto objectId = LRU.front(); - LRU.pop_front(); - - auto maybeFile = Files.find(objectId); - if (maybeFile != Files.end()) { - auto path = GetDir(objectId) + "/" + maybeFile->second.Name; - - YQL_LOG(DEBUG) << "Remove File " << path << " UsedSize " << ToString(UsedSize) << " FileSize " << ToString(maybeFile->second.Size); - - UsedSize -= maybeFile->second.Size; - NFs::Remove(path); - Files.erase(maybeFile); - } - } -} - -TString TFileCache::GetDir(const TString& md5) const -{ - return BaseDir + "/" + md5.substr(0, md5.length() / 2); -} - -void TFileCache::AddFile(const TString& path, const TString& objectId) -{ - auto dir = GetDir(objectId); - NFs::MakeDirectoryRecursive(dir, NFs::FP_NONSECRET_FILE, false); - - auto newBaseName = objectId.substr(objectId.length() / 2); - auto newName = dir + "/" + newBaseName; -#ifndef _win_ - chmod(path.c_str(), 0755); -#endif - TFileObject file; - file.LastAccess = 0; // unused - file.Size = TFile(path, RdOnly).GetLength(); - file.Name = newBaseName; - file.ObjectId = objectId; - - { - TGuard<TMutex> guard(Mutex); - LRU.push_back(objectId); - file.Position = --LRU.end(); - UsedSize += file.Size; - Files.insert(std::make_pair(objectId, file)); - } - - // don't lock on fs - NFs::Rename(path, newName); - - TGuard<TMutex> guard(Mutex); - Clean(); -} - -TMaybe<TString> TFileCache::FindFile(const TString& objectId) -{ - TString fileName; - { - TGuard<TMutex> guard(Mutex); - auto maybeFile = Files.find(objectId); - if (maybeFile == Files.end()) { - return { }; - } - LRU.erase(maybeFile->second.Position); - LRU.push_back(objectId); - maybeFile->second.Position = --LRU.end(); - fileName = maybeFile->second.Name; - } - - return {GetDir(objectId) + "/" + fileName}; -} - -bool TFileCache::Contains(const TString& objectId) -{ - TGuard<TMutex> guard(Mutex); - return Files.contains(objectId); -} - -void TFileCache::Walk(const std::function<void(const TString& objectId)>& f) -{ - TGuard<TMutex> guard(Mutex); - for (const auto& objectId : LRU) { - f(objectId); - } -} - -} // namespace NYql - + +#include <util/system/fs.h> +#include <util/system/file.h> +#include <util/folder/path.h> +#include <util/folder/iterator.h> + +namespace NYql { + +TFileCache::TFileCache(const TString& baseDir, i64 size) + : BaseDir(baseDir) + , TotalSize(size) +{ + NFs::MakeDirectoryRecursive(BaseDir, NFs::FP_NONSECRET_FILE, false); + + BaseDir = TFsPath(BaseDir).RealPath().GetPath(); + + Scan(); +} + +void TFileCache::Scan() +{ + TDirIterator it( + BaseDir, + TDirIterator::TOptions() + .SetMaxLevel(2)); + + TVector<TFileObject> allFiles; + for (const auto& item : it) { + if (item.fts_level != 2) { + continue; + } + TFsPath path = item.fts_path; + if (!path.IsFile()) { + continue; + } + auto split = path.PathSplit(); + + TFileObject file; + file.ObjectId = TString(split[split.size() - 2]) + TString(split[split.size() - 1]); + file.Name = TString(split[split.size() - 1]); +#ifdef _linux_ + file.LastAccess = item.fts_statp->st_atim.tv_sec; +#endif + file.Size = item.fts_statp->st_size; + + UsedSize += file.Size; + + YQL_LOG(DEBUG) << file.Name << "|" << file.ObjectId; + + allFiles.emplace_back(std::move(file)); + } + + std::sort(allFiles.begin(), allFiles.end(), [](const auto&a, const auto& b) { + return a.LastAccess < b.LastAccess; + }); + + TGuard<TMutex> guard(Mutex); + for (auto& file : allFiles) { + LRU.push_back(file.ObjectId); + file.Position = --LRU.end(); + Files.insert(std::make_pair(file.ObjectId, file)); + } + + Clean(); +} + +ui64 TFileCache::FreeDiskSize() { + return TotalSize - UsedSize; +} + +ui64 TFileCache::UsedDiskSize() { + return UsedSize; +} + +void TFileCache::Clean() { + while (UsedSize > TotalSize && LRU.size() > 2) { + auto objectId = LRU.front(); + LRU.pop_front(); + + auto maybeFile = Files.find(objectId); + if (maybeFile != Files.end()) { + auto path = GetDir(objectId) + "/" + maybeFile->second.Name; + + YQL_LOG(DEBUG) << "Remove File " << path << " UsedSize " << ToString(UsedSize) << " FileSize " << ToString(maybeFile->second.Size); + + UsedSize -= maybeFile->second.Size; + NFs::Remove(path); + Files.erase(maybeFile); + } + } +} + +TString TFileCache::GetDir(const TString& md5) const +{ + return BaseDir + "/" + md5.substr(0, md5.length() / 2); +} + +void TFileCache::AddFile(const TString& path, const TString& objectId) +{ + auto dir = GetDir(objectId); + NFs::MakeDirectoryRecursive(dir, NFs::FP_NONSECRET_FILE, false); + + auto newBaseName = objectId.substr(objectId.length() / 2); + auto newName = dir + "/" + newBaseName; +#ifndef _win_ + chmod(path.c_str(), 0755); +#endif + TFileObject file; + file.LastAccess = 0; // unused + file.Size = TFile(path, RdOnly).GetLength(); + file.Name = newBaseName; + file.ObjectId = objectId; + + { + TGuard<TMutex> guard(Mutex); + LRU.push_back(objectId); + file.Position = --LRU.end(); + UsedSize += file.Size; + Files.insert(std::make_pair(objectId, file)); + } + + // don't lock on fs + NFs::Rename(path, newName); + + TGuard<TMutex> guard(Mutex); + Clean(); +} + +TMaybe<TString> TFileCache::FindFile(const TString& objectId) +{ + TString fileName; + { + TGuard<TMutex> guard(Mutex); + auto maybeFile = Files.find(objectId); + if (maybeFile == Files.end()) { + return { }; + } + LRU.erase(maybeFile->second.Position); + LRU.push_back(objectId); + maybeFile->second.Position = --LRU.end(); + fileName = maybeFile->second.Name; + } + + return {GetDir(objectId) + "/" + fileName}; +} + +bool TFileCache::Contains(const TString& objectId) +{ + TGuard<TMutex> guard(Mutex); + return Files.contains(objectId); +} + +void TFileCache::Walk(const std::function<void(const TString& objectId)>& f) +{ + TGuard<TMutex> guard(Mutex); + for (const auto& objectId : LRU) { + f(objectId); + } +} + +} // namespace NYql + diff --git a/ydb/library/yql/providers/dq/runtime/file_cache.h b/ydb/library/yql/providers/dq/runtime/file_cache.h index 00edb7925e..bb1c9bf9ae 100644 --- a/ydb/library/yql/providers/dq/runtime/file_cache.h +++ b/ydb/library/yql/providers/dq/runtime/file_cache.h @@ -1,52 +1,52 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/task_runner/file_cache.h> - -#include <util/generic/list.h> - -namespace NYql { - -class TFileCache: public IFileCache { -public: - explicit TFileCache(const TString& baseDir, i64 size); - - void AddFile(const TString& path, const TString& md5) override; - - TMaybe<TString> FindFile(const TString& md5) override; - - void Walk(const std::function<void(const TString& objectId)>&) override; - - ui64 FreeDiskSize() override; - - ui64 UsedDiskSize() override; - - bool Contains(const TString& objectId) override; - - TString GetDir() override { - return BaseDir; - }; - -private: - TString GetDir(const TString& md5) const; - - void Scan(); - void Clean(); - - TString BaseDir; - - struct TFileObject { - TString ObjectId; - TString Name; - ui64 Size = 0; - ui64 LastAccess = 0; - TList<TString>::iterator Position; - }; - TList<TString> LRU; - THashMap<TString, TFileObject> Files; - ui64 UsedSize = 0; - ui64 TotalSize = 0; - - TMutex Mutex; -}; - -} // namespace NYql + +#include <util/generic/list.h> + +namespace NYql { + +class TFileCache: public IFileCache { +public: + explicit TFileCache(const TString& baseDir, i64 size); + + void AddFile(const TString& path, const TString& md5) override; + + TMaybe<TString> FindFile(const TString& md5) override; + + void Walk(const std::function<void(const TString& objectId)>&) override; + + ui64 FreeDiskSize() override; + + ui64 UsedDiskSize() override; + + bool Contains(const TString& objectId) override; + + TString GetDir() override { + return BaseDir; + }; + +private: + TString GetDir(const TString& md5) const; + + void Scan(); + void Clean(); + + TString BaseDir; + + struct TFileObject { + TString ObjectId; + TString Name; + ui64 Size = 0; + ui64 LastAccess = 0; + TList<TString>::iterator Position; + }; + TList<TString> LRU; + THashMap<TString, TFileObject> Files; + ui64 UsedSize = 0; + ui64 TotalSize = 0; + + TMutex Mutex; +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/runtime/runtime_data.cpp b/ydb/library/yql/providers/dq/runtime/runtime_data.cpp index 21afe52d76..c4c8113bae 100644 --- a/ydb/library/yql/providers/dq/runtime/runtime_data.cpp +++ b/ydb/library/yql/providers/dq/runtime/runtime_data.cpp @@ -1,84 +1,84 @@ -#include "runtime_data.h" - -namespace NYql { - -void TWorkerRuntimeData::AddRusageDelta(const TRusage& delta) { - auto now = TInstant::Now(); - TGuard<TAdaptiveLock> guard(Lock); - if (!LastUpdate) { - Rusages[Index] = delta; - } else { - auto seconds = (now - LastUpdate).Seconds(); - // Current : Rusage[Index+seconds] - // Prev: Rusage[Index..Index+seconds-1] - for (ui32 i = 1; i <= seconds; i=i+1) { - auto prev = Rusages[(Index + i - 1) % Rusages.size()]; - Rusages[(Index + i) % Rusages.size()] = prev; - } - Index = (Index + seconds) % Rusages.size(); - if (!RusageFull && (Index + 1) == static_cast<int>(Rusages.size())) { - RusageFull = true; - } - Rusages[Index].Stime += delta.Stime; - Rusages[Index].Utime += delta.Utime; - Rusages[Index].MajorPageFaults += delta.MajorPageFaults; - } - LastUpdate = TInstant::Seconds(now.Seconds()); -} - -TRusage TWorkerRuntimeData::GetRusageDelta() { - TGuard<TAdaptiveLock> guard(Lock); - TRusage cur, prev; - cur = Rusages[Index]; - if (RusageFull) { - prev = Rusages[(Index + 1) % Rusages.size()]; - } else { - prev = Rusages[0]; - } - cur.Stime -= prev.Stime; - cur.Utime -= prev.Utime; - cur.MajorPageFaults -= prev.MajorPageFaults; - return cur; -} - -TRusage TWorkerRuntimeData::GetRusage() { - TGuard<TAdaptiveLock> guard(Lock); - return Rusages[Index]; -} - -TDuration TWorkerRuntimeData::GetRusageWindow() { - if (RusageFull) { - return TDuration::Seconds(WindowSize); - } else { - TGuard<TAdaptiveLock> guard(Lock); - return Max(TDuration::Seconds(1), TDuration::Seconds(Index)); - } -} - -void TWorkerRuntimeData::OnWorkerStart(const TString& operationId) { - TGuard<TAdaptiveLock> guard(WorkersLock); - RunningWorkers[operationId]++; -} - -void TWorkerRuntimeData::OnWorkerStop(const TString& operationId) { - TGuard<TAdaptiveLock> guard(WorkersLock); - auto it = RunningWorkers.find(operationId); - if (it != RunningWorkers.end() && ! --it->second) { - RunningWorkers.erase(it); - } -} - -THashMap<TString,int> TWorkerRuntimeData::GetRunningWorkers() const { - TGuard<TAdaptiveLock> guard(WorkersLock); - return RunningWorkers; -} - -void TWorkerRuntimeData::UpdateChannelInputDelay(TDuration duration) { - Y_UNUSED(duration); -} - -void TWorkerRuntimeData::UpdateChannelOutputDelay(TDuration duration) { - Y_UNUSED(duration); -} - -} // namespace NYql +#include "runtime_data.h" + +namespace NYql { + +void TWorkerRuntimeData::AddRusageDelta(const TRusage& delta) { + auto now = TInstant::Now(); + TGuard<TAdaptiveLock> guard(Lock); + if (!LastUpdate) { + Rusages[Index] = delta; + } else { + auto seconds = (now - LastUpdate).Seconds(); + // Current : Rusage[Index+seconds] + // Prev: Rusage[Index..Index+seconds-1] + for (ui32 i = 1; i <= seconds; i=i+1) { + auto prev = Rusages[(Index + i - 1) % Rusages.size()]; + Rusages[(Index + i) % Rusages.size()] = prev; + } + Index = (Index + seconds) % Rusages.size(); + if (!RusageFull && (Index + 1) == static_cast<int>(Rusages.size())) { + RusageFull = true; + } + Rusages[Index].Stime += delta.Stime; + Rusages[Index].Utime += delta.Utime; + Rusages[Index].MajorPageFaults += delta.MajorPageFaults; + } + LastUpdate = TInstant::Seconds(now.Seconds()); +} + +TRusage TWorkerRuntimeData::GetRusageDelta() { + TGuard<TAdaptiveLock> guard(Lock); + TRusage cur, prev; + cur = Rusages[Index]; + if (RusageFull) { + prev = Rusages[(Index + 1) % Rusages.size()]; + } else { + prev = Rusages[0]; + } + cur.Stime -= prev.Stime; + cur.Utime -= prev.Utime; + cur.MajorPageFaults -= prev.MajorPageFaults; + return cur; +} + +TRusage TWorkerRuntimeData::GetRusage() { + TGuard<TAdaptiveLock> guard(Lock); + return Rusages[Index]; +} + +TDuration TWorkerRuntimeData::GetRusageWindow() { + if (RusageFull) { + return TDuration::Seconds(WindowSize); + } else { + TGuard<TAdaptiveLock> guard(Lock); + return Max(TDuration::Seconds(1), TDuration::Seconds(Index)); + } +} + +void TWorkerRuntimeData::OnWorkerStart(const TString& operationId) { + TGuard<TAdaptiveLock> guard(WorkersLock); + RunningWorkers[operationId]++; +} + +void TWorkerRuntimeData::OnWorkerStop(const TString& operationId) { + TGuard<TAdaptiveLock> guard(WorkersLock); + auto it = RunningWorkers.find(operationId); + if (it != RunningWorkers.end() && ! --it->second) { + RunningWorkers.erase(it); + } +} + +THashMap<TString,int> TWorkerRuntimeData::GetRunningWorkers() const { + TGuard<TAdaptiveLock> guard(WorkersLock); + return RunningWorkers; +} + +void TWorkerRuntimeData::UpdateChannelInputDelay(TDuration duration) { + Y_UNUSED(duration); +} + +void TWorkerRuntimeData::UpdateChannelOutputDelay(TDuration duration) { + Y_UNUSED(duration); +} + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/runtime/runtime_data.h b/ydb/library/yql/providers/dq/runtime/runtime_data.h index d7aea5b0d2..bbca8a5506 100644 --- a/ydb/library/yql/providers/dq/runtime/runtime_data.h +++ b/ydb/library/yql/providers/dq/runtime/runtime_data.h @@ -1,46 +1,46 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/generic/guid.h> -#include <util/generic/vector.h> -#include <util/generic/hash_set.h> -#include <util/system/rusage.h> -#include <util/system/spinlock.h> - -#include <atomic> - -namespace NYql { - -struct TWorkerRuntimeData { - std::atomic<ui32> Epoch = 0; - - TGUID WorkerId; - TString ClusterName; - - void AddRusageDelta(const TRusage& delta); - - TDuration GetRusageWindow(); - TRusage GetRusageDelta(); - TRusage GetRusage(); - - void UpdateChannelInputDelay(TDuration duration); - void UpdateChannelOutputDelay(TDuration duration); - - void OnWorkerStart(const TString& operationId); - void OnWorkerStop(const TString& operationId); - - THashMap<TString,int> GetRunningWorkers() const; - -private: - TAdaptiveLock Lock; - const int WindowSize = 60; - TVector<TRusage> Rusages = TVector<TRusage>(WindowSize); - TInstant LastUpdate; - std::atomic<bool> RusageFull = false; - int Index = 0; - - TAdaptiveLock WorkersLock; - THashMap<TString,int> RunningWorkers; -}; - -} // namespace NYql +#pragma once + +#include <util/generic/string.h> +#include <util/generic/guid.h> +#include <util/generic/vector.h> +#include <util/generic/hash_set.h> +#include <util/system/rusage.h> +#include <util/system/spinlock.h> + +#include <atomic> + +namespace NYql { + +struct TWorkerRuntimeData { + std::atomic<ui32> Epoch = 0; + + TGUID WorkerId; + TString ClusterName; + + void AddRusageDelta(const TRusage& delta); + + TDuration GetRusageWindow(); + TRusage GetRusageDelta(); + TRusage GetRusage(); + + void UpdateChannelInputDelay(TDuration duration); + void UpdateChannelOutputDelay(TDuration duration); + + void OnWorkerStart(const TString& operationId); + void OnWorkerStop(const TString& operationId); + + THashMap<TString,int> GetRunningWorkers() const; + +private: + TAdaptiveLock Lock; + const int WindowSize = 60; + TVector<TRusage> Rusages = TVector<TRusage>(WindowSize); + TInstant LastUpdate; + std::atomic<bool> RusageFull = false; + int Index = 0; + + TAdaptiveLock WorkersLock; + THashMap<TString,int> RunningWorkers; +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp index 80c9c7a853..9b41920b17 100644 --- a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp +++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp @@ -7,54 +7,54 @@ #include <ydb/library/yql/providers/dq/api/protos/dqs.pb.h> #include <ydb/library/yql/providers/dq/api/protos/task_command_executor.pb.h> #include <ydb/library/yql/utils/backtrace/backtrace.h> - + #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> #include <ydb/library/yql/minikql/mkql_node_serialization.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node.h> -#include <ydb/library/yql/minikql/mkql_string_util.h> - +#include <ydb/library/yql/minikql/mkql_string_util.h> + #include <ydb/library/yql/minikql/aligned_page_pool.h> #include <ydb/library/yql/dq/runtime/dq_input_channel.h> #include <ydb/library/yql/dq/runtime/dq_output_channel.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/yql_panic.h> - -#include <util/system/thread.h> -#include <util/system/fs.h> -#include <util/system/env.h> -#include <util/system/rusage.h> - -#include <util/folder/path.h> - -#include <util/stream/file.h> -#include <util/stream/pipe.h> - + +#include <util/system/thread.h> +#include <util/system/fs.h> +#include <util/system/env.h> +#include <util/system/rusage.h> + +#include <util/folder/path.h> + +#include <util/stream/file.h> +#include <util/stream/pipe.h> + #ifdef _unix_ -#include <sys/resource.h> -#endif - -namespace NYql { -namespace NTaskRunnerProxy { - -// static const int CurrentProtocolVersion = 1; -// static const int CurrentProtocolVersion = 2; // GetFreeSpace +#include <sys/resource.h> +#endif + +namespace NYql { +namespace NTaskRunnerProxy { + +// static const int CurrentProtocolVersion = 1; +// static const int CurrentProtocolVersion = 2; // GetFreeSpace // static const int CurrentProtocolVersion = 3; // Calls for ComputeActor -// static const int CurrentProtocolVersion = 4; // Calls for Sources -static const int CurrentProtocolVersion = 5; // Calls for Sinks - -template<typename T> -void ToProto(T* s1, const NDq::TDqInputChannelStats* ss) -{ - s1->SetChannelId(ss->ChannelId); - s1->SetChunks(ss->Chunks); - s1->SetBytes(ss->Bytes); - s1->SetRowsIn(ss->RowsIn); - s1->SetRowsOut(ss->RowsOut); - s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); +// static const int CurrentProtocolVersion = 4; // Calls for Sources +static const int CurrentProtocolVersion = 5; // Calls for Sinks + +template<typename T> +void ToProto(T* s1, const NDq::TDqInputChannelStats* ss) +{ + s1->SetChannelId(ss->ChannelId); + s1->SetChunks(ss->Chunks); + s1->SetBytes(ss->Bytes); + s1->SetRowsIn(ss->RowsIn); + s1->SetRowsOut(ss->RowsOut); + s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); s1->SetDeserializationTimeUs(ss->DeserializationTime.MicroSeconds()); -} - -template<typename T> +} + +template<typename T> void ToProto(T* s1, const NDq::TDqSourceStats* ss) { s1->SetChunks(ss->Chunks); @@ -66,113 +66,113 @@ void ToProto(T* s1, const NDq::TDqSourceStats* ss) } template<typename T> -void ToProto(T* s1, const NDq::TDqOutputChannelStats* ss) -{ - s1->SetChannelId(ss->ChannelId); - s1->SetChunks(ss->Chunks); - s1->SetBytes(ss->Bytes); - s1->SetRowsIn(ss->RowsIn); - s1->SetRowsOut(ss->RowsOut); - s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); -} - -template<typename T> -void ToProto(T* s1, const NDq::TDqSinkStats* ss) -{ - s1->SetChunks(ss->Chunks); - s1->SetBytes(ss->Bytes); - s1->SetRowsIn(ss->RowsIn); - s1->SetRowsOut(ss->RowsOut); - s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); -} - -class TTaskCommandExecutor { -public: - TTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError) +void ToProto(T* s1, const NDq::TDqOutputChannelStats* ss) +{ + s1->SetChannelId(ss->ChannelId); + s1->SetChunks(ss->Chunks); + s1->SetBytes(ss->Bytes); + s1->SetRowsIn(ss->RowsIn); + s1->SetRowsOut(ss->RowsOut); + s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); +} + +template<typename T> +void ToProto(T* s1, const NDq::TDqSinkStats* ss) +{ + s1->SetChunks(ss->Chunks); + s1->SetBytes(ss->Bytes); + s1->SetRowsIn(ss->RowsIn); + s1->SetRowsOut(ss->RowsOut); + s1->SetMaxMemoryUsage(ss->MaxMemoryUsage); +} + +class TTaskCommandExecutor { +public: + TTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError) : ComputationFactory(std::move(compFactory)) , TaskTransformFactory(std::move(taskTransformFactory)) - , JobStats(std::move(jobStats)) + , JobStats(std::move(jobStats)) , TerminateOnError(terminateOnError) { } - - void UpdateStats() { - if (!Runner) { - return; - } - - if (JobStats) { + + void UpdateStats() { + if (!Runner) { + return; + } + + if (JobStats) { auto taskId = Runner->GetTaskId(); - std::map<TString, TString> labels = { - {"Task", ToString(taskId)} - }; - - JobStats->ForEachStat([&](const NKikimr::NMiniKQL::TStatKey& key, i64 value) { - if (value) { - TString name = TString(key.GetName()); - auto counterName = TCounters::GetCounterName( - "TaskRunner", - labels, - name); - auto& old = CurrentJobStats[counterName]; - if (name.EndsWith("Time")) { - QueryStat.AddTimeCounter(counterName, value - old); - } else { - QueryStat.AddCounter(counterName, value - old); - } - old = value; - } - }); - } - - QueryStat.AddTaskRunnerStats( - *Runner->GetStats(), - CurrentStats, - Runner->GetTaskId()); - } - - void UpdateOutputChannelStats(ui64 channelId) - { - if (!Runner) { - return; - } - auto s = Runner->GetStats(); - auto maybeOutputChannelStats = s->OutputChannels.find(channelId); - if (maybeOutputChannelStats == s->OutputChannels.end() || !maybeOutputChannelStats->second) { - return; - } - auto maybeOutputChannelOldStats = CurrentOutputChannelsStats.find(channelId); - if (maybeOutputChannelOldStats == CurrentOutputChannelsStats.end()) { - maybeOutputChannelOldStats = CurrentOutputChannelsStats.emplace_hint( - maybeOutputChannelOldStats, channelId, NDq::TDqOutputChannelStats(channelId)); - } - QueryStat.AddOutputChannelStats( - *maybeOutputChannelStats->second, - maybeOutputChannelOldStats->second, - Runner->GetTaskId(), channelId); - } - - void UpdateInputChannelStats(ui64 channelId) - { - if (!Runner) { - return; - } - auto s = Runner->GetStats(); - auto maybeInputChannelStats = s->InputChannels.find(channelId); - if (maybeInputChannelStats == s->InputChannels.end() || !maybeInputChannelStats->second) { - return; - } - auto maybeInputChannelOldStats = CurrentInputChannelsStats.find(channelId); - if (maybeInputChannelOldStats == CurrentInputChannelsStats.end()) { - maybeInputChannelOldStats = CurrentInputChannelsStats.emplace_hint( - maybeInputChannelOldStats, channelId, NDq::TDqInputChannelStats(channelId)); - } - QueryStat.AddInputChannelStats( - *maybeInputChannelStats->second, - maybeInputChannelOldStats->second, - Runner->GetTaskId(), channelId); - } - + std::map<TString, TString> labels = { + {"Task", ToString(taskId)} + }; + + JobStats->ForEachStat([&](const NKikimr::NMiniKQL::TStatKey& key, i64 value) { + if (value) { + TString name = TString(key.GetName()); + auto counterName = TCounters::GetCounterName( + "TaskRunner", + labels, + name); + auto& old = CurrentJobStats[counterName]; + if (name.EndsWith("Time")) { + QueryStat.AddTimeCounter(counterName, value - old); + } else { + QueryStat.AddCounter(counterName, value - old); + } + old = value; + } + }); + } + + QueryStat.AddTaskRunnerStats( + *Runner->GetStats(), + CurrentStats, + Runner->GetTaskId()); + } + + void UpdateOutputChannelStats(ui64 channelId) + { + if (!Runner) { + return; + } + auto s = Runner->GetStats(); + auto maybeOutputChannelStats = s->OutputChannels.find(channelId); + if (maybeOutputChannelStats == s->OutputChannels.end() || !maybeOutputChannelStats->second) { + return; + } + auto maybeOutputChannelOldStats = CurrentOutputChannelsStats.find(channelId); + if (maybeOutputChannelOldStats == CurrentOutputChannelsStats.end()) { + maybeOutputChannelOldStats = CurrentOutputChannelsStats.emplace_hint( + maybeOutputChannelOldStats, channelId, NDq::TDqOutputChannelStats(channelId)); + } + QueryStat.AddOutputChannelStats( + *maybeOutputChannelStats->second, + maybeOutputChannelOldStats->second, + Runner->GetTaskId(), channelId); + } + + void UpdateInputChannelStats(ui64 channelId) + { + if (!Runner) { + return; + } + auto s = Runner->GetStats(); + auto maybeInputChannelStats = s->InputChannels.find(channelId); + if (maybeInputChannelStats == s->InputChannels.end() || !maybeInputChannelStats->second) { + return; + } + auto maybeInputChannelOldStats = CurrentInputChannelsStats.find(channelId); + if (maybeInputChannelOldStats == CurrentInputChannelsStats.end()) { + maybeInputChannelOldStats = CurrentInputChannelsStats.emplace_hint( + maybeInputChannelOldStats, channelId, NDq::TDqInputChannelStats(channelId)); + } + QueryStat.AddInputChannelStats( + *maybeInputChannelStats->second, + maybeInputChannelOldStats->second, + Runner->GetTaskId(), channelId); + } + void UpdateSourceStats(ui64 inputIndex) { if (!Runner) { @@ -180,125 +180,125 @@ public: } auto s = Runner->GetStats(); auto maybeSourceStats = s->Sources.find(inputIndex); - if (maybeSourceStats == s->Sources.end() || !maybeSourceStats->second) { + if (maybeSourceStats == s->Sources.end() || !maybeSourceStats->second) { return; } - auto maybeSourceOldStats = CurrentSourcesStats.find(inputIndex); - if (maybeSourceOldStats == CurrentSourcesStats.end()) { - maybeSourceOldStats = CurrentSourcesStats.emplace_hint( - maybeSourceOldStats, inputIndex, NDq::TDqSourceStats(inputIndex)); - } - QueryStat.AddSourceStats( - *maybeSourceStats->second, - maybeSourceOldStats->second, - Runner->GetTaskId(), inputIndex); + auto maybeSourceOldStats = CurrentSourcesStats.find(inputIndex); + if (maybeSourceOldStats == CurrentSourcesStats.end()) { + maybeSourceOldStats = CurrentSourcesStats.emplace_hint( + maybeSourceOldStats, inputIndex, NDq::TDqSourceStats(inputIndex)); + } + QueryStat.AddSourceStats( + *maybeSourceStats->second, + maybeSourceOldStats->second, + Runner->GetTaskId(), inputIndex); } - template<typename T> - void UpdateStats(T& t) { - UpdateStats(); - QueryStat.FlushCounters(t); - - auto currentRusage = TRusage::Get(); - TRusage delta; - delta.Utime = currentRusage.Utime - Rusage.Utime; - delta.Stime = currentRusage.Stime - Rusage.Stime; - delta.MajorPageFaults = currentRusage.MajorPageFaults - Rusage.MajorPageFaults; - t.MutableRusage()->SetUtime(delta.Utime.MicroSeconds()); - t.MutableRusage()->SetStime(delta.Stime.MicroSeconds()); - t.MutableRusage()->SetMajorPageFaults(delta.MajorPageFaults); - Rusage = currentRusage; - } - - void Run() - { - try { - RunUnsafe(); - } catch (const std::exception& ex) { - try { - // don't generate core if parent died before child - Cerr << ex.what() << Endl; - Cerr << "Command " << LastCommand << Endl; - Cerr << "Version " << LastVersion << Endl; - Cerr << "TaskId " << LastTaskId << Endl; - Cerr << "ChannelId " << LastChannelId << Endl; - Cerr << "CommandNo " << CommandNo << Endl; - UpdateStats(); - - // sum min max avg count - for (const auto& [k, v] : QueryStat.Get()) { - Cerr << "Counter: " - << k << " " - << v.Sum << " " - << v.Min << " " - << v.Max << " " - << v.Avg << " " - << v.Count << Endl; - } - } catch (...) { } - } - _exit(127); - } - - void RunUnsafe() - { - FunctionRegistry = QueryStat.Measure<TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry>>("CreateFunctionRegistry", [&]() { - auto ret = NKikimr::NMiniKQL::CreateFunctionRegistry( - &KikimrBackTrace, NKikimr::NMiniKQL::CreateBuiltinRegistry(), false, {})->Clone(); - NKikimr::NMiniKQL::FillStaticModules(*ret); - return ret; - }); - - auto deterministicMode = !!GetEnv("YQL_DETERMINISTIC_MODE"); - - auto randomProvider = deterministicMode - ? CreateDeterministicRandomProvider(1) - : CreateDefaultRandomProvider(); - auto timeProvider = deterministicMode - ? CreateDeterministicTimeProvider(10000000) - : CreateDefaultTimeProvider(); - - Ctx.FuncRegistry = FunctionRegistry.Get(); - Ctx.ComputationFactory = ComputationFactory; - Ctx.RandomProvider = randomProvider.Get(); - Ctx.TimeProvider = timeProvider.Get(); - - TPipedInput input(0); - TPipedOutput output(1); - - while (true) { - NDqProto::TCommandHeader header; - header.Load(&input); - - ui64 taskId = header.GetTaskId(); - ui64 channelId = header.GetChannelId(); - - LastCommand = static_cast<i64>(header.GetCommand()); - LastVersion = header.GetVersion(); - LastTaskId = taskId; - LastChannelId = channelId; - CommandNo ++; - - switch (header.GetCommand()) { - case NDqProto::TCommandHeader::VERSION: { - NDqProto::TGetVersionResponse response; - response.SetVersion(CurrentProtocolVersion); - response.Save(&output); - break; - } - case NDqProto::TCommandHeader::PUSH: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + template<typename T> + void UpdateStats(T& t) { + UpdateStats(); + QueryStat.FlushCounters(t); + + auto currentRusage = TRusage::Get(); + TRusage delta; + delta.Utime = currentRusage.Utime - Rusage.Utime; + delta.Stime = currentRusage.Stime - Rusage.Stime; + delta.MajorPageFaults = currentRusage.MajorPageFaults - Rusage.MajorPageFaults; + t.MutableRusage()->SetUtime(delta.Utime.MicroSeconds()); + t.MutableRusage()->SetStime(delta.Stime.MicroSeconds()); + t.MutableRusage()->SetMajorPageFaults(delta.MajorPageFaults); + Rusage = currentRusage; + } + + void Run() + { + try { + RunUnsafe(); + } catch (const std::exception& ex) { + try { + // don't generate core if parent died before child + Cerr << ex.what() << Endl; + Cerr << "Command " << LastCommand << Endl; + Cerr << "Version " << LastVersion << Endl; + Cerr << "TaskId " << LastTaskId << Endl; + Cerr << "ChannelId " << LastChannelId << Endl; + Cerr << "CommandNo " << CommandNo << Endl; + UpdateStats(); + + // sum min max avg count + for (const auto& [k, v] : QueryStat.Get()) { + Cerr << "Counter: " + << k << " " + << v.Sum << " " + << v.Min << " " + << v.Max << " " + << v.Avg << " " + << v.Count << Endl; + } + } catch (...) { } + } + _exit(127); + } + + void RunUnsafe() + { + FunctionRegistry = QueryStat.Measure<TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry>>("CreateFunctionRegistry", [&]() { + auto ret = NKikimr::NMiniKQL::CreateFunctionRegistry( + &KikimrBackTrace, NKikimr::NMiniKQL::CreateBuiltinRegistry(), false, {})->Clone(); + NKikimr::NMiniKQL::FillStaticModules(*ret); + return ret; + }); + + auto deterministicMode = !!GetEnv("YQL_DETERMINISTIC_MODE"); + + auto randomProvider = deterministicMode + ? CreateDeterministicRandomProvider(1) + : CreateDefaultRandomProvider(); + auto timeProvider = deterministicMode + ? CreateDeterministicTimeProvider(10000000) + : CreateDefaultTimeProvider(); + + Ctx.FuncRegistry = FunctionRegistry.Get(); + Ctx.ComputationFactory = ComputationFactory; + Ctx.RandomProvider = randomProvider.Get(); + Ctx.TimeProvider = timeProvider.Get(); + + TPipedInput input(0); + TPipedOutput output(1); + + while (true) { + NDqProto::TCommandHeader header; + header.Load(&input); + + ui64 taskId = header.GetTaskId(); + ui64 channelId = header.GetChannelId(); + + LastCommand = static_cast<i64>(header.GetCommand()); + LastVersion = header.GetVersion(); + LastTaskId = taskId; + LastChannelId = channelId; + CommandNo ++; + + switch (header.GetCommand()) { + case NDqProto::TCommandHeader::VERSION: { + NDqProto::TGetVersionResponse response; + response.SetVersion(CurrentProtocolVersion); + response.Save(&output); + break; + } + case NDqProto::TCommandHeader::PUSH: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto channel = Runner->GetInputChannel(channelId); - - NDqProto::TData data; - data.Load(&input); - + auto channel = Runner->GetInputChannel(channelId); + + NDqProto::TData data; + data.Load(&input); + auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - channel->Push(std::move(data)); - UpdateInputChannelStats(channelId); - break; - } + channel->Push(std::move(data)); + UpdateInputChannelStats(channelId); + break; + } case NDqProto::TCommandHeader::PUSH_SOURCE: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -328,53 +328,53 @@ public: NDq::TDqDataSerializer dataSerializer(Runner->GetTypeEnv(), Runner->GetHolderFactory(), transportVersion); NKikimr::NMiniKQL::TUnboxedValueVector buffer; buffer.reserve(request.GetData().GetRows()); - if (request.GetString().empty() && request.GetChunks() == 0) { - dataSerializer.Deserialize(request.GetData(), source->GetInputType(), buffer); - } else if (!request.GetString().empty()) { - buffer.reserve(request.GetString().size()); - for (auto& row : request.GetString()) { - buffer.emplace_back(NKikimr::NMiniKQL::MakeString(row)); - } - } else { - i64 chunks = request.GetChunks(); - buffer.reserve(chunks); - for (i64 i = 0; i < chunks; i++) { - NDqProto::TSourcePushChunk chunk; - chunk.Load(&input); - i64 parts = chunk.GetParts(); - - if (parts == 1) { - buffer.emplace_back(NKikimr::NMiniKQL::MakeString(chunk.GetString())); - } else { - TString str; - for (i64 j = 0; j < parts; j++) { - NDqProto::TSourcePushPart part; - part.Load(&input); - str += part.GetString(); - } - buffer.emplace_back(NKikimr::NMiniKQL::MakeString(str)); - } - } - } + if (request.GetString().empty() && request.GetChunks() == 0) { + dataSerializer.Deserialize(request.GetData(), source->GetInputType(), buffer); + } else if (!request.GetString().empty()) { + buffer.reserve(request.GetString().size()); + for (auto& row : request.GetString()) { + buffer.emplace_back(NKikimr::NMiniKQL::MakeString(row)); + } + } else { + i64 chunks = request.GetChunks(); + buffer.reserve(chunks); + for (i64 i = 0; i < chunks; i++) { + NDqProto::TSourcePushChunk chunk; + chunk.Load(&input); + i64 parts = chunk.GetParts(); + + if (parts == 1) { + buffer.emplace_back(NKikimr::NMiniKQL::MakeString(chunk.GetString())); + } else { + TString str; + for (i64 j = 0; j < parts; j++) { + NDqProto::TSourcePushPart part; + part.Load(&input); + str += part.GetString(); + } + buffer.emplace_back(NKikimr::NMiniKQL::MakeString(str)); + } + } + } source->Push(std::move(buffer), request.GetSpace()); - UpdateSourceStats(channelId); + UpdateSourceStats(channelId); break; } - case NDqProto::TCommandHeader::FINISH: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::FINISH: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - Runner->GetInputChannel(channelId)->Finish(); - UpdateInputChannelStats(channelId); - break; - } - case NDqProto::TCommandHeader::FINISH_OUTPUT: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + Runner->GetInputChannel(channelId)->Finish(); + UpdateInputChannelStats(channelId); + break; + } + case NDqProto::TCommandHeader::FINISH_OUTPUT: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - Runner->GetOutputChannel(channelId)->Finish(); - UpdateOutputChannelStats(channelId); - break; - } + Runner->GetOutputChannel(channelId)->Finish(); + UpdateOutputChannelStats(channelId); + break; + } case NDqProto::TCommandHeader::FINISH_SOURCE: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -382,33 +382,33 @@ public: UpdateSourceStats(channelId); break; } - case NDqProto::TCommandHeader::DROP_OUTPUT: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::DROP_OUTPUT: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto res = Runner->GetOutputChannel(channelId)->Drop(); - - NDqProto::TDropOutputResponse response; - response.SetResult(res); - response.Save(&output); - break; - } - case NDqProto::TCommandHeader::TERMINATE_OUTPUT: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + auto res = Runner->GetOutputChannel(channelId)->Drop(); + + NDqProto::TDropOutputResponse response; + response.SetResult(res); + response.Save(&output); + break; + } + case NDqProto::TCommandHeader::TERMINATE_OUTPUT: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - Runner->GetOutputChannel(channelId)->Terminate(); - UpdateOutputChannelStats(channelId); - break; - } - case NDqProto::TCommandHeader::GET_STORED_BYTES: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + Runner->GetOutputChannel(channelId)->Terminate(); + UpdateOutputChannelStats(channelId); + break; + } + case NDqProto::TCommandHeader::GET_STORED_BYTES: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto res = Runner->GetInputChannel(channelId)->GetStoredBytes(); - - NDqProto::TGetStoredBytesResponse response; - response.SetResult(res); - response.Save(&output); - break; - } + auto res = Runner->GetInputChannel(channelId)->GetStoredBytes(); + + NDqProto::TGetStoredBytesResponse response; + response.SetResult(res); + response.Save(&output); + break; + } case NDqProto::TCommandHeader::GET_STORED_BYTES_SOURCE: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); @@ -419,81 +419,81 @@ public: response.Save(&output); break; } - case NDqProto::TCommandHeader::POP: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + case NDqProto::TCommandHeader::POP: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - + Y_ENSURE(taskId == Runner->GetTaskId()); - - auto channel = Runner->GetOutputChannel(channelId); - - NDqProto::TPopResponse response; - + + auto channel = Runner->GetOutputChannel(channelId); + + NDqProto::TPopResponse response; + response.SetResult(channel->Pop(*response.MutableData(), 5 << 20)); - UpdateOutputChannelStats(channelId); - QueryStat.FlushCounters(response); - response.Save(&output); - - break; - } - case NDqProto::TCommandHeader::IS_FINISHED: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + UpdateOutputChannelStats(channelId); + QueryStat.FlushCounters(response); + response.Save(&output); + + break; + } + case NDqProto::TCommandHeader::IS_FINISHED: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); Y_ENSURE(taskId == Runner->GetTaskId()); - auto channel = Runner->GetOutputChannel(channelId); - NDqProto::TIsFinishedResponse response; - response.SetResult(channel->IsFinished()); - response.Save(&output); - break; - } - case NDqProto::TCommandHeader::RUN: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + auto channel = Runner->GetOutputChannel(channelId); + NDqProto::TIsFinishedResponse response; + response.SetResult(channel->IsFinished()); + response.Save(&output); + break; + } + case NDqProto::TCommandHeader::RUN: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0)); - + Y_ENSURE(taskId == Runner->GetTaskId()); try { NDqProto::TRunResponse response; auto status = Runner->Run(); response.SetResult(static_cast<ui32>(status)); - UpdateStats(response); + UpdateStats(response); response.Save(&output); } catch (const NKikimr::TMemoryLimitExceededException& ex) { - throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit"; - } - break; - } - case NDqProto::TCommandHeader::PREPARE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); - - NDqProto::TPrepareRequest request; - Yql::DqsProto::TTaskMeta taskMeta; - - QueryStat.Measure<void>("LoadTask", [&]() { - request.Load(&input); - }); - - request.GetTask().GetMeta().UnpackTo(&taskMeta); + throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit"; + } + break; + } + case NDqProto::TCommandHeader::PREPARE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + + NDqProto::TPrepareRequest request; + Yql::DqsProto::TTaskMeta taskMeta; + + QueryStat.Measure<void>("LoadTask", [&]() { + request.Load(&input); + }); + + request.GetTask().GetMeta().UnpackTo(&taskMeta); try { Prepare(request.GetTask(), taskMeta, output); } catch (const NKikimr::TMemoryLimitExceededException& ex) { throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit"; } - break; - } - case NDqProto::TCommandHeader::GET_INPUT_TYPE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + break; + } + case NDqProto::TCommandHeader::GET_INPUT_TYPE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - + Y_ENSURE(taskId == Runner->GetTaskId()); - auto channel = Runner->GetInputChannel(channelId); - auto inputType = channel->GetInputType(); - + auto channel = Runner->GetInputChannel(channelId); + auto inputType = channel->GetInputType(); + NDqProto::TGetTypeResponse response; - response.SetResult(NKikimr::NMiniKQL::SerializeNode(inputType, Runner->GetTypeEnv())); - response.Save(&output); - - break; - } + response.SetResult(NKikimr::NMiniKQL::SerializeNode(inputType, Runner->GetTypeEnv())); + response.Save(&output); + + break; + } case NDqProto::TCommandHeader::GET_SOURCE_TYPE: { Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit @@ -508,19 +508,19 @@ public: break; } - case NDqProto::TCommandHeader::GET_FREE_SPACE: { - Y_ENSURE(header.GetVersion() >= 2); - + case NDqProto::TCommandHeader::GET_FREE_SPACE: { + Y_ENSURE(header.GetVersion() >= 2); + Y_ENSURE(taskId == Runner->GetTaskId()); - auto channel = Runner->GetInputChannel(channelId); - NDqProto::TGetFreeSpaceResponse response; - response.SetFreeSpace(channel->GetFreeSpace()); - response.Save(&output); - - break; - } + auto channel = Runner->GetInputChannel(channelId); + NDqProto::TGetFreeSpaceResponse response; + response.SetFreeSpace(channel->GetFreeSpace()); + response.Save(&output); + + break; + } case NDqProto::TCommandHeader::GET_FREE_SPACE_SOURCE: { - Y_ENSURE(header.GetVersion() >= 4); + Y_ENSURE(header.GetVersion() >= 4); Y_ENSURE(taskId == Runner->GetTaskId()); auto source = Runner->GetSource(channelId); @@ -530,14 +530,14 @@ public: break; } - case NDqProto::TCommandHeader::GET_STATS: { - Y_ENSURE(header.GetVersion() >= 3); + case NDqProto::TCommandHeader::GET_STATS: { + Y_ENSURE(header.GetVersion() >= 3); Y_ENSURE(taskId == Runner->GetTaskId()); - auto stats = Runner->GetStats(); - - NDqProto::TGetStatsResponse response; - auto* s = response.MutableStats(); - s->SetTaskId(taskId); + auto stats = Runner->GetStats(); + + NDqProto::TGetStatsResponse response; + auto* s = response.MutableStats(); + s->SetTaskId(taskId); //s->SetStartTs(stats->StartTs.MilliSeconds()); //s->SetFinishTs(stats->FinishTs.MilliSeconds()); s->SetBuildCpuTimeUs(stats->BuildCpuTime.MicroSeconds()); @@ -552,7 +552,7 @@ public: //s->SetTotalTime(stats->TotalTime.MilliSeconds()); s->SetWaitTimeUs(stats->WaitTime.MicroSeconds()); s->SetWaitOutputTimeUs(stats->WaitOutputTime.MicroSeconds()); - + //s->SetMkqlTotalNodes(stats->MkqlTotalNodes); //s->SetMkqlCodegenFunctions(stats->MkqlCodegenFunctions); //s->SetCodeGenTotalInstructions(stats->CodeGenTotalInstructions); @@ -560,34 +560,34 @@ public: //s->SetCodeGenFullTime(stats->CodeGenFullTime); //s->SetCodeGenFinalizeTime(stats->CodeGenFinalizeTime); //s->SetCodeGenModulePassTime(stats->CodeGenModulePassTime); - - for (const auto& [id, ss] : stats->OutputChannels) { - ToProto(s->AddOutputChannels(), ss); - } - - for (const auto& [id, ss] : stats->InputChannels) { - ToProto(s->AddInputChannels(), ss); - } + + for (const auto& [id, ss] : stats->OutputChannels) { + ToProto(s->AddOutputChannels(), ss); + } + + for (const auto& [id, ss] : stats->InputChannels) { + ToProto(s->AddInputChannels(), ss); + } for (const auto& [id, ss] : stats->Sources) { ToProto(s->AddSources(), ss); } - response.Save(&output); - - break; - } - case NDqProto::TCommandHeader::GET_STATS_INPUT: { - Y_ENSURE(header.GetVersion() >= 3); + response.Save(&output); + + break; + } + case NDqProto::TCommandHeader::GET_STATS_INPUT: { + Y_ENSURE(header.GetVersion() >= 3); Y_ENSURE(taskId == Runner->GetTaskId()); - auto ss = Runner->GetInputChannel(channelId)->GetStats(); - NDqProto::TGetStatsInputResponse response; - ToProto(response.MutableStats(), ss); - response.Save(&output); - break; - } + auto ss = Runner->GetInputChannel(channelId)->GetStats(); + NDqProto::TGetStatsInputResponse response; + ToProto(response.MutableStats(), ss); + response.Save(&output); + break; + } case NDqProto::TCommandHeader::GET_STATS_SOURCE: { - Y_ENSURE(header.GetVersion() >= 4); + Y_ENSURE(header.GetVersion() >= 4); Y_ENSURE(taskId == Runner->GetTaskId()); auto ss = Runner->GetSource(channelId)->GetStats(); NDqProto::TGetStatsSourceResponse response; @@ -595,91 +595,91 @@ public: response.Save(&output); break; } - case NDqProto::TCommandHeader::GET_STATS_OUTPUT: { - Y_ENSURE(header.GetVersion() >= 3); + case NDqProto::TCommandHeader::GET_STATS_OUTPUT: { + Y_ENSURE(header.GetVersion() >= 3); + Y_ENSURE(taskId == Runner->GetTaskId()); + auto ss = Runner->GetOutputChannel(channelId)->GetStats(); + NDqProto::TGetStatsOutputResponse response; + ToProto(response.MutableStats(), ss); + response.Save(&output); + break; + } + // Sinks + case NDqProto::TCommandHeader::SINK_IS_FINISHED: { + Y_ENSURE(header.GetVersion() >= 5); + Y_ENSURE(taskId == Runner->GetTaskId()); + auto flag = Runner->GetSink(channelId)->IsFinished(); + NDqProto::TIsFinishedResponse response; + response.SetResult(flag); + response.Save(&output); + break; + } + case NDqProto::TCommandHeader::SINK_POP: { + Y_ENSURE(header.GetVersion() >= 5); Y_ENSURE(taskId == Runner->GetTaskId()); - auto ss = Runner->GetOutputChannel(channelId)->GetStats(); - NDqProto::TGetStatsOutputResponse response; - ToProto(response.MutableStats(), ss); - response.Save(&output); - break; - } - // Sinks - case NDqProto::TCommandHeader::SINK_IS_FINISHED: { - Y_ENSURE(header.GetVersion() >= 5); - Y_ENSURE(taskId == Runner->GetTaskId()); - auto flag = Runner->GetSink(channelId)->IsFinished(); - NDqProto::TIsFinishedResponse response; - response.SetResult(flag); - response.Save(&output); - break; - } - case NDqProto::TCommandHeader::SINK_POP: { - Y_ENSURE(header.GetVersion() >= 5); - Y_ENSURE(taskId == Runner->GetTaskId()); - - NDqProto::TSinkPopRequest request; - request.Load(&input); - + + NDqProto::TSinkPopRequest request; + request.Load(&input); + auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - NKikimr::NMiniKQL::TUnboxedValueVector batch; - auto sink = Runner->GetSink(channelId); - auto* outputType = sink->GetOutputType(); - auto bytes = sink->Pop(batch, request.GetBytes()); - - NDqProto::TSinkPopResponse response; - if (request.GetRaw()) { - for (auto& raw : batch) { - *response.AddString() = raw.AsStringRef(); - } - } else { - NDq::TDqDataSerializer dataSerializer( - Runner->GetTypeEnv(), - Runner->GetHolderFactory(), - NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); - - *response.MutableData() = dataSerializer.Serialize( - batch, - static_cast<NKikimr::NMiniKQL::TType*>(outputType)); - } - response.SetBytes(bytes); - response.Save(&output); - break; - } - case NDqProto::TCommandHeader::SINK_OUTPUT_TYPE: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + NKikimr::NMiniKQL::TUnboxedValueVector batch; + auto sink = Runner->GetSink(channelId); + auto* outputType = sink->GetOutputType(); + auto bytes = sink->Pop(batch, request.GetBytes()); + + NDqProto::TSinkPopResponse response; + if (request.GetRaw()) { + for (auto& raw : batch) { + *response.AddString() = raw.AsStringRef(); + } + } else { + NDq::TDqDataSerializer dataSerializer( + Runner->GetTypeEnv(), + Runner->GetHolderFactory(), + NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); + + *response.MutableData() = dataSerializer.Serialize( + batch, + static_cast<NKikimr::NMiniKQL::TType*>(outputType)); + } + response.SetBytes(bytes); + response.Save(&output); + break; + } + case NDqProto::TCommandHeader::SINK_OUTPUT_TYPE: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit - - Y_ENSURE(taskId == Runner->GetTaskId()); - auto outputType = Runner->GetSink(channelId)->GetOutputType(); - - NDqProto::TGetTypeResponse response; - response.SetResult(NKikimr::NMiniKQL::SerializeNode(outputType, Runner->GetTypeEnv())); - response.Save(&output); - - break; - } - case NDqProto::TCommandHeader::SINK_STATS: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); - Y_ENSURE(taskId == Runner->GetTaskId()); - - auto* stats = Runner->GetSink(channelId)->GetStats(); - NDqProto::TSinkStatsResponse response; - ToProto(response.MutableStats(), stats); - response.Save(&output); - break; - } - // - case NDqProto::TCommandHeader::STOP: { - Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); - _exit(0); - } - default: - Y_VERIFY(false); - } - } - } - + + Y_ENSURE(taskId == Runner->GetTaskId()); + auto outputType = Runner->GetSink(channelId)->GetOutputType(); + + NDqProto::TGetTypeResponse response; + response.SetResult(NKikimr::NMiniKQL::SerializeNode(outputType, Runner->GetTypeEnv())); + response.Save(&output); + + break; + } + case NDqProto::TCommandHeader::SINK_STATS: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + Y_ENSURE(taskId == Runner->GetTaskId()); + + auto* stats = Runner->GetSink(channelId)->GetStats(); + NDqProto::TSinkStatsResponse response; + ToProto(response.MutableStats(), stats); + response.Save(&output); + break; + } + // + case NDqProto::TCommandHeader::STOP: { + Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion); + _exit(0); + } + default: + Y_VERIFY(false); + } + } + } + void DontCollectDumps() { #ifdef _unix_ rlimit limit = {0, 0}; @@ -689,112 +689,112 @@ public: #endif } - template<typename T> - void Prepare(const NDqProto::TDqTask& task, const T& taskMeta, TPipedOutput& output) { - NYql::NDqProto::TPrepareResponse result; - result.SetResult(true); // COMPAT(aozeritsky) YQL-14268 - - DqConfiguration->Dispatch(taskMeta.GetSettings()); - DqConfiguration->FreezeDefaults(); - if (!DqConfiguration->CollectCoreDumps.Get().GetOrElse(false)) { - DontCollectDumps(); - } - // TODO: Maybe use taskParams from task.GetTask().GetParameters() - THashMap<TString, TString> taskParams; - for (const auto& x: taskMeta.GetTaskParams()) { - taskParams[x.first] = x.second; - } - - TString workingDirectory = taskParams[NTaskRunnerProxy::WorkingDirectoryParamName]; - Y_VERIFY(workingDirectory); - NFs::SetCurrentWorkingDirectory(workingDirectory); - - THashMap<TString, TString> modulesMapping; - - QueryStat.Measure<void>("LoadUdfs", [&]() - { - for (const auto& file : taskMeta.GetFiles()) { - auto name = file.GetName(); - auto path = file.GetLocalPath(); - - switch (file.GetObjectType()) { - case Yql::DqsProto::TFile::EEXE_FILE: - modulesMapping.emplace("EXE", file.GetObjectId()); - break; - case Yql::DqsProto::TFile::EUSER_FILE: - break; - case Yql::DqsProto::TFile::EUDF_FILE: - FunctionRegistry->LoadUdfs(path, EmptyRemappings, 0); - modulesMapping.emplace(TFsPath(path).RealPath(), file.GetObjectId()); - break; - default: - Y_VERIFY(false); - } - } - }); - - NBacktrace::SetModulesMapping(modulesMapping); - - QueryStat.Measure<void>("MakeDqTaskRunner", [&]() { - NDq::TDqTaskRunnerSettings settings; - settings.CollectBasicStats = true; - settings.CollectProfileStats = true; - settings.TerminateOnError = TerminateOnError; + template<typename T> + void Prepare(const NDqProto::TDqTask& task, const T& taskMeta, TPipedOutput& output) { + NYql::NDqProto::TPrepareResponse result; + result.SetResult(true); // COMPAT(aozeritsky) YQL-14268 + + DqConfiguration->Dispatch(taskMeta.GetSettings()); + DqConfiguration->FreezeDefaults(); + if (!DqConfiguration->CollectCoreDumps.Get().GetOrElse(false)) { + DontCollectDumps(); + } + // TODO: Maybe use taskParams from task.GetTask().GetParameters() + THashMap<TString, TString> taskParams; + for (const auto& x: taskMeta.GetTaskParams()) { + taskParams[x.first] = x.second; + } + + TString workingDirectory = taskParams[NTaskRunnerProxy::WorkingDirectoryParamName]; + Y_VERIFY(workingDirectory); + NFs::SetCurrentWorkingDirectory(workingDirectory); + + THashMap<TString, TString> modulesMapping; + + QueryStat.Measure<void>("LoadUdfs", [&]() + { + for (const auto& file : taskMeta.GetFiles()) { + auto name = file.GetName(); + auto path = file.GetLocalPath(); + + switch (file.GetObjectType()) { + case Yql::DqsProto::TFile::EEXE_FILE: + modulesMapping.emplace("EXE", file.GetObjectId()); + break; + case Yql::DqsProto::TFile::EUSER_FILE: + break; + case Yql::DqsProto::TFile::EUDF_FILE: + FunctionRegistry->LoadUdfs(path, EmptyRemappings, 0); + modulesMapping.emplace(TFsPath(path).RealPath(), file.GetObjectId()); + break; + default: + Y_VERIFY(false); + } + } + }); + + NBacktrace::SetModulesMapping(modulesMapping); + + QueryStat.Measure<void>("MakeDqTaskRunner", [&]() { + NDq::TDqTaskRunnerSettings settings; + settings.CollectBasicStats = true; + settings.CollectProfileStats = true; + settings.TerminateOnError = TerminateOnError; settings.AllowGeneratorsInUnboxedValues = true; - for (const auto& x: taskMeta.GetSecureParams()) { - settings.SecureParams[x.first] = x.second; - YQL_LOG(DEBUG) << "SecureParam " << x.first << ":XXX"; - } - settings.OptLLVM = DqConfiguration->OptLLVM.Get().GetOrElse(""); - - Ctx.FuncProvider = TaskTransformFactory(taskParams, Ctx.FuncRegistry); - + for (const auto& x: taskMeta.GetSecureParams()) { + settings.SecureParams[x.first] = x.second; + YQL_LOG(DEBUG) << "SecureParam " << x.first << ":XXX"; + } + settings.OptLLVM = DqConfiguration->OptLLVM.Get().GetOrElse(""); + + Ctx.FuncProvider = TaskTransformFactory(taskParams, Ctx.FuncRegistry); + Runner = MakeDqTaskRunner(Ctx, settings, nullptr); - }); - + }); + auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0)); - QueryStat.Measure<void>("Prepare", [&]() { - NDq::TDqTaskRunnerMemoryLimits limits; - limits.ChannelBufferSize = DqConfiguration->ChannelBufferSize.Get().GetOrElse(2000_MB); - limits.OutputChunkMaxSize = DqConfiguration->OutputChunkMaxSize.Get().GetOrElse(4_MB); + QueryStat.Measure<void>("Prepare", [&]() { + NDq::TDqTaskRunnerMemoryLimits limits; + limits.ChannelBufferSize = DqConfiguration->ChannelBufferSize.Get().GetOrElse(2000_MB); + limits.OutputChunkMaxSize = DqConfiguration->OutputChunkMaxSize.Get().GetOrElse(4_MB); Runner->Prepare(task, limits); - }); - - result.Save(&output); - } - + }); + + result.Save(&output); + } + NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory; TTaskTransformFactory TaskTransformFactory; - THashMap<TString, i64> CurrentJobStats; - NKikimr::NMiniKQL::IStatsRegistry* JobStats; + THashMap<TString, i64> CurrentJobStats; + NKikimr::NMiniKQL::IStatsRegistry* JobStats; bool TerminateOnError; - TIntrusivePtr<NDq::IDqTaskRunner> Runner; - TCounters QueryStat; - TDqConfiguration::TPtr DqConfiguration = MakeIntrusive<TDqConfiguration>(); - TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FunctionRegistry; - NDq::TDqTaskRunnerContext Ctx; - const NKikimr::NMiniKQL::TUdfModuleRemappings EmptyRemappings; - - TRusage Rusage; - - NDq::TDqTaskRunnerStats CurrentStats; - std::unordered_map<ui64, NDq::TDqInputChannelStats> CurrentInputChannelsStats; - std::unordered_map<ui64, NDq::TDqSourceStats> CurrentSourcesStats; - std::unordered_map<ui64, NDq::TDqOutputChannelStats> CurrentOutputChannelsStats; - - i64 LastCommand = -1; - i64 LastVersion = -1; - ui64 LastTaskId = -1; - ui64 LastChannelId = -1; - i64 CommandNo = 0; -}; - -int CreateTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError) -{ - TTaskCommandExecutor(compFactory, taskTransformFactory, jobStats, terminateOnError).Run(); - return 0; -} - -} // namespace NTaskRunnerProxy -} // namespace NYql + TIntrusivePtr<NDq::IDqTaskRunner> Runner; + TCounters QueryStat; + TDqConfiguration::TPtr DqConfiguration = MakeIntrusive<TDqConfiguration>(); + TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FunctionRegistry; + NDq::TDqTaskRunnerContext Ctx; + const NKikimr::NMiniKQL::TUdfModuleRemappings EmptyRemappings; + + TRusage Rusage; + + NDq::TDqTaskRunnerStats CurrentStats; + std::unordered_map<ui64, NDq::TDqInputChannelStats> CurrentInputChannelsStats; + std::unordered_map<ui64, NDq::TDqSourceStats> CurrentSourcesStats; + std::unordered_map<ui64, NDq::TDqOutputChannelStats> CurrentOutputChannelsStats; + + i64 LastCommand = -1; + i64 LastVersion = -1; + ui64 LastTaskId = -1; + ui64 LastChannelId = -1; + i64 CommandNo = 0; +}; + +int CreateTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError) +{ + TTaskCommandExecutor(compFactory, taskTransformFactory, jobStats, terminateOnError).Run(); + return 0; +} + +} // namespace NTaskRunnerProxy +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.h b/ydb/library/yql/providers/dq/runtime/task_command_executor.h index ba85fc0549..75f7c60177 100644 --- a/ydb/library/yql/providers/dq/runtime/task_command_executor.h +++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.h @@ -7,7 +7,7 @@ namespace NYql { namespace NTaskRunnerProxy { -int CreateTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError = false); +int CreateTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError = false); } // namespace NTaskRunnerProxy } // namespace NYql diff --git a/ydb/library/yql/providers/dq/runtime/ya.make b/ydb/library/yql/providers/dq/runtime/ya.make index b1cfead065..cab6308d76 100644 --- a/ydb/library/yql/providers/dq/runtime/ya.make +++ b/ydb/library/yql/providers/dq/runtime/ya.make @@ -1,8 +1,8 @@ -LIBRARY() - +LIBRARY() + OWNER(g:yql) - -PEERDIR( + +PEERDIR( ydb/library/yql/minikql ydb/library/yql/minikql/computation ydb/library/yql/minikql/invoke_builtins @@ -18,14 +18,14 @@ PEERDIR( ydb/library/yql/providers/dq/counters ydb/library/yql/providers/dq/interface ydb/library/yql/providers/dq/task_runner -) - +) + YQL_LAST_ABI_VERSION() -SRCS( - file_cache.cpp - task_command_executor.cpp - runtime_data.cpp -) - -END() +SRCS( + file_cache.cpp + task_command_executor.cpp + runtime_data.cpp +) + +END() diff --git a/ydb/library/yql/providers/dq/service/grpc_service.cpp b/ydb/library/yql/providers/dq/service/grpc_service.cpp index ca8c25893c..65a9e500d9 100644 --- a/ydb/library/yql/providers/dq/service/grpc_service.cpp +++ b/ydb/library/yql/providers/dq/service/grpc_service.cpp @@ -1,7 +1,7 @@ #include "grpc_service.h" #include <ydb/library/yql/utils/log/log.h> - + #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> #include <ydb/library/yql/providers/dq/actors/executer_actor.h> #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> @@ -10,17 +10,17 @@ #include <ydb/library/yql/providers/dq/actors/events.h> #include <ydb/library/yql/providers/dq/actors/task_controller.h> #include <ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h> - + #include <ydb/library/yql/providers/dq/counters/counters.h> #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> #include <ydb/library/yql/providers/dq/common/yql_dq_common.h> - -//#include <yql/tools/yqlworker/dq/worker_manager/benchmark.h> - + +//#include <yql/tools/yqlworker/dq/worker_manager/benchmark.h> + #include <ydb/library/yql/public/issue/yql_issue_message.h> - + #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> - + #include <library/cpp/grpc/server/grpc_counters.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> @@ -35,9 +35,9 @@ namespace NYql::NDqs { using namespace NYql::NDqs; using namespace NKikimr; - using namespace NThreading; - using namespace NMonitoring; - using namespace NActors; + using namespace NThreading; + using namespace NMonitoring; + using namespace NActors; namespace { NGrpc::ICounterBlockPtr BuildCB(TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, const TString& name) { @@ -55,110 +55,110 @@ namespace NYql::NDqs { grpcCB); } - template<typename RequestType, typename ResponseType> + template<typename RequestType, typename ResponseType> class TServiceProxyActor: public TSynchronizableRichActor<TServiceProxyActor<RequestType, ResponseType>> { public: - static constexpr char ActorName[] = "SERVICE_PROXY"; - - explicit TServiceProxyActor( + static constexpr char ActorName[] = "SERVICE_PROXY"; + + explicit TServiceProxyActor( NGrpc::IRequestContextBase* ctx, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, - const TString& traceId, const TString& username) + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TString& traceId, const TString& username) : TSynchronizableRichActor<TServiceProxyActor<RequestType, ResponseType>>(&TServiceProxyActor::Handler) - , Ctx(ctx) - , Counters(counters) - , ServiceProxyActorCounters(counters->GetSubgroup("component", "ServiceProxyActor")) - , ClientDisconnectedCounter(ServiceProxyActorCounters->GetCounter("ClientDisconnected", /*derivative=*/ true)) - , RetryCounter(ServiceProxyActorCounters->GetCounter("Retry", /*derivative=*/ true)) - , FallbackCounter(ServiceProxyActorCounters->GetCounter("Fallback", /*derivative=*/ true)) - , ErrorCounter(ServiceProxyActorCounters->GetCounter("UnrecoverableError", /*derivative=*/ true)) - , Request(dynamic_cast<const RequestType*>(ctx->GetRequest())) - , TraceId(traceId) + , Ctx(ctx) + , Counters(counters) + , ServiceProxyActorCounters(counters->GetSubgroup("component", "ServiceProxyActor")) + , ClientDisconnectedCounter(ServiceProxyActorCounters->GetCounter("ClientDisconnected", /*derivative=*/ true)) + , RetryCounter(ServiceProxyActorCounters->GetCounter("Retry", /*derivative=*/ true)) + , FallbackCounter(ServiceProxyActorCounters->GetCounter("Fallback", /*derivative=*/ true)) + , ErrorCounter(ServiceProxyActorCounters->GetCounter("UnrecoverableError", /*derivative=*/ true)) + , Request(dynamic_cast<const RequestType*>(ctx->GetRequest())) + , TraceId(traceId) , Username(username) - , Promise(NewPromise<void>()) + , Promise(NewPromise<void>()) { - Settings->Dispatch(Request->GetSettings()); - Settings->FreezeDefaults(); - - MaxRetries = Settings->MaxRetries.Get().GetOrElse(MaxRetries); - RetryBackoff = TDuration::MilliSeconds(Settings->RetryBackoffMs.Get().GetOrElse(RetryBackoff.MilliSeconds())); + Settings->Dispatch(Request->GetSettings()); + Settings->FreezeDefaults(); + + MaxRetries = Settings->MaxRetries.Get().GetOrElse(MaxRetries); + RetryBackoff = TDuration::MilliSeconds(Settings->RetryBackoffMs.Get().GetOrElse(RetryBackoff.MilliSeconds())); } STRICT_STFUNC(Handler, { HFunc(TEvQueryResponse, OnReturnResult); - cFunc(TEvents::TEvPoison::EventType, OnPoison); - SFunc(TEvents::TEvBootstrap, DoBootstrap) + cFunc(TEvents::TEvPoison::EventType, OnPoison); + SFunc(TEvents::TEvBootstrap, DoBootstrap) }) TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { - return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0); + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0); } - void OnPoison() { + void OnPoison() { YQL_LOG_CTX_SCOPE(TraceId); YQL_LOG(DEBUG) << __FUNCTION__ ; - ReplyError(grpc::UNAVAILABLE, "Unexpected error"); - *ClientDisconnectedCounter += 1; - } - - void DoPassAway() override { - Promise.SetValue(); - } - - void DoBootstrap(const NActors::TActorContext& ctx) { + ReplyError(grpc::UNAVAILABLE, "Unexpected error"); + *ClientDisconnectedCounter += 1; + } + + void DoPassAway() override { + Promise.SetValue(); + } + + void DoBootstrap(const NActors::TActorContext& ctx) { YQL_LOG_CTX_SCOPE(TraceId); - if (!CtxSubscribed) { - auto selfId = ctx.SelfID; - auto* actorSystem = ctx.ExecutorThread.ActorSystem; + if (!CtxSubscribed) { + auto selfId = ctx.SelfID; + auto* actorSystem = ctx.ExecutorThread.ActorSystem; Ctx->GetFinishFuture().Subscribe([selfId, actorSystem](const NGrpc::IRequestContextBase::TAsyncFinishResult& future) { - Y_VERIFY(future.HasValue()); + Y_VERIFY(future.HasValue()); if (future.GetValue() == NGrpc::IRequestContextBase::EFinishStatus::CANCEL) { - actorSystem->Send(selfId, new TEvents::TEvPoison()); - } - }); - CtxSubscribed = true; - } - Bootstrap(); - } - - virtual void Bootstrap() = 0; - - void SendResponse(TEvQueryResponse::TPtr& ev) - { - auto& result = ev->Get()->Record; + actorSystem->Send(selfId, new TEvents::TEvPoison()); + } + }); + CtxSubscribed = true; + } + Bootstrap(); + } + + virtual void Bootstrap() = 0; + + void SendResponse(TEvQueryResponse::TPtr& ev) + { + auto& result = ev->Get()->Record; Yql::DqsProto::ExecuteQueryResult queryResult; queryResult.Mutableresult()->CopyFrom(result.resultset()); - queryResult.set_yson(result.yson()); - - if (result.GetNeedFallback()) { - NYql::TIssue rootIssue("Fatal Error"); - rootIssue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR); - NYql::TIssues issues; - NYql::IssuesFromMessage(result.GetIssues(), issues); - for (const auto& issue: issues) { - rootIssue.AddSubIssue(MakeIntrusive<TIssue>(issue)); - } - result.MutableIssues()->Clear(); - NYql::IssuesToMessage({rootIssue}, result.MutableIssues()); - } - - for (const auto& [k, v] : QueryStat.Get()) { - std::map<TString, TString> labels; - TString prefix, name; + queryResult.set_yson(result.yson()); + + if (result.GetNeedFallback()) { + NYql::TIssue rootIssue("Fatal Error"); + rootIssue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR); + NYql::TIssues issues; + NYql::IssuesFromMessage(result.GetIssues(), issues); + for (const auto& issue: issues) { + rootIssue.AddSubIssue(MakeIntrusive<TIssue>(issue)); + } + result.MutableIssues()->Clear(); + NYql::IssuesToMessage({rootIssue}, result.MutableIssues()); + } + + for (const auto& [k, v] : QueryStat.Get()) { + std::map<TString, TString> labels; + TString prefix, name; if (NCommon::ParseCounterName(&prefix, &labels, &name, k)) { - if (prefix == "Actor") { - auto group = Counters->GetSubgroup("counters", "Actor"); - for (const auto& [k, v] : labels) { - group = group->GetSubgroup(k, v); - } - group->GetHistogram(name, ExponentialHistogram(10, 2, 50))->Collect(v.Sum); - } - } - } + if (prefix == "Actor") { + auto group = Counters->GetSubgroup("counters", "Actor"); + for (const auto& [k, v] : labels) { + group = group->GetSubgroup(k, v); + } + group->GetHistogram(name, ExponentialHistogram(10, 2, 50))->Collect(v.Sum); + } + } + } auto aggregatedQueryStat = AggregateQueryStatsByStage(QueryStat, Task2Stage); - + aggregatedQueryStat.FlushCounters(ResponseBuffer); - + auto& operation = *ResponseBuffer.mutable_operation(); operation.Setready(true); operation.Mutableresult()->PackFrom(queryResult); @@ -166,113 +166,113 @@ namespace NYql::NDqs { ResponseBuffer.SetTruncated(result.GetTruncated()); Reply(Ydb::StatusIds::SUCCESS, result.GetIssues().size() > 0); - } - + } + virtual void DoRetry() - { - this->CleanupChildren(); - auto selfId = this->SelfId(); - TActivationContext::Schedule(RetryBackoff, new IEventHandle(selfId, selfId, new TEvents::TEvBootstrap(), 0)); - Retry += 1; - *RetryCounter +=1 ; - } - - void OnReturnResult(TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { + { + this->CleanupChildren(); + auto selfId = this->SelfId(); + TActivationContext::Schedule(RetryBackoff, new IEventHandle(selfId, selfId, new TEvents::TEvBootstrap(), 0)); + Retry += 1; + *RetryCounter +=1 ; + } + + void OnReturnResult(TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { Y_UNUSED(ctx); - YQL_LOG_CTX_SCOPE(TraceId); - YQL_LOG(DEBUG) << "TServiceProxyActor::OnReturnResult " << ev->Get()->Record.GetMetric().size(); - QueryStat.AddCounters(ev->Get()->Record); - if (ev->Get()->Record.GetIssues().size() > 0 && ev->Get()->Record.GetRetriable() && Retry < MaxRetries) { - QueryStat.AddCounter("Retry", TDuration::MilliSeconds(0)); - NYql::TIssues issues; - NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); - YQL_LOG(WARN) << "Retry " << Retry << " Issues: " << issues.ToString(); + YQL_LOG_CTX_SCOPE(TraceId); + YQL_LOG(DEBUG) << "TServiceProxyActor::OnReturnResult " << ev->Get()->Record.GetMetric().size(); + QueryStat.AddCounters(ev->Get()->Record); + if (ev->Get()->Record.GetIssues().size() > 0 && ev->Get()->Record.GetRetriable() && Retry < MaxRetries) { + QueryStat.AddCounter("Retry", TDuration::MilliSeconds(0)); + NYql::TIssues issues; + NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); + YQL_LOG(WARN) << "Retry " << Retry << " Issues: " << issues.ToString(); DoRetry(); - } else { - if (ev->Get()->Record.GetIssues().size() > 0) { - NYql::TIssues issues; - NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); - YQL_LOG(WARN) << "Issues: " << issues.ToString(); - *ErrorCounter += 1; - } - if (ev->Get()->Record.GetNeedFallback()) { - // TODO: Remove GetNeedFallback, use only issue codes! - *FallbackCounter += 1; - } - SendResponse(ev); - } - } - - TFuture<void> GetFuture() { - return Promise.GetFuture(); - } - + } else { + if (ev->Get()->Record.GetIssues().size() > 0) { + NYql::TIssues issues; + NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); + YQL_LOG(WARN) << "Issues: " << issues.ToString(); + *ErrorCounter += 1; + } + if (ev->Get()->Record.GetNeedFallback()) { + // TODO: Remove GetNeedFallback, use only issue codes! + *FallbackCounter += 1; + } + SendResponse(ev); + } + } + + TFuture<void> GetFuture() { + return Promise.GetFuture(); + } + virtual void ReplyError(grpc::StatusCode code, const TString& msg) { - Ctx->ReplyError(code, msg); - this->PassAway(); - } - + Ctx->ReplyError(code, msg); + this->PassAway(); + } + virtual void Reply(ui32 status, bool hasIssues) { Y_UNUSED(hasIssues); Ctx->Reply(&ResponseBuffer, status); - this->PassAway(); - } - - private: + this->PassAway(); + } + + private: NGrpc::IRequestContextBase* Ctx; - bool CtxSubscribed = false; + bool CtxSubscribed = false; ResponseType ResponseBuffer; - - protected: - TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; - TIntrusivePtr<NMonitoring::TDynamicCounters> ServiceProxyActorCounters; - TDynamicCounters::TCounterPtr ClientDisconnectedCounter; - TDynamicCounters::TCounterPtr RetryCounter; - TDynamicCounters::TCounterPtr FallbackCounter; - TDynamicCounters::TCounterPtr ErrorCounter; - - const RequestType* Request; - const TString TraceId; + + protected: + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + TIntrusivePtr<NMonitoring::TDynamicCounters> ServiceProxyActorCounters; + TDynamicCounters::TCounterPtr ClientDisconnectedCounter; + TDynamicCounters::TCounterPtr RetryCounter; + TDynamicCounters::TCounterPtr FallbackCounter; + TDynamicCounters::TCounterPtr ErrorCounter; + + const RequestType* Request; + const TString TraceId; const TString Username; - TPromise<void> Promise; - const TInstant RequestStartTime = TInstant::Now(); - - TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); - - int Retry = 0; - int MaxRetries = 10; - TDuration RetryBackoff = TDuration::MilliSeconds(1000); - - NYql::TCounters QueryStat; - THashMap<ui64, ui64> Task2Stage; + TPromise<void> Promise; + const TInstant RequestStartTime = TInstant::Now(); + + TDqConfiguration::TPtr Settings = MakeIntrusive<TDqConfiguration>(); + + int Retry = 0; + int MaxRetries = 10; + TDuration RetryBackoff = TDuration::MilliSeconds(1000); + + NYql::TCounters QueryStat; + THashMap<ui64, ui64> Task2Stage; void RestoreRequest() { Request = dynamic_cast<const RequestType*>(Ctx->GetRequest()); } }; - class TExecuteGraphProxyActor: public TServiceProxyActor<Yql::DqsProto::ExecuteGraphRequest, Yql::DqsProto::ExecuteGraphResponse> { - public: + class TExecuteGraphProxyActor: public TServiceProxyActor<Yql::DqsProto::ExecuteGraphRequest, Yql::DqsProto::ExecuteGraphResponse> { + public: using TBase = TServiceProxyActor<Yql::DqsProto::ExecuteGraphRequest, Yql::DqsProto::ExecuteGraphResponse>; TExecuteGraphProxyActor(NGrpc::IRequestContextBase* ctx, - const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, + const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters, const TString& traceId, const TString& username, const NActors::TActorId& graphExecutionEventsActorId) - : TServiceProxyActor(ctx, counters, traceId, username) + : TServiceProxyActor(ctx, counters, traceId, username) , GraphExecutionEventsActorId(graphExecutionEventsActorId) { } - + void DoRetry() override { YQL_LOG(DEBUG) << __FUNCTION__; SendEvent(NYql::NDqProto::EGraphExecutionEventType::FAIL, nullptr, [this](const auto& ev) { - if (ev->Get()->Record.GetErrorMessage()) { - TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage()); - } else { - RestoreRequest(); - ModifiedRequest.Reset(); - TBase::DoRetry(); - } + if (ev->Get()->Record.GetErrorMessage()) { + TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage()); + } else { + RestoreRequest(); + ModifiedRequest.Reset(); + TBase::DoRetry(); + } }); } @@ -282,10 +282,10 @@ namespace NYql::NDqs { : NYql::NDqProto::EGraphExecutionEventType::SUCCESS; SendEvent(eventType, nullptr, [this, status, hasIssues](const auto& ev) { if (!hasIssues && ev->Get()->Record.GetErrorMessage()) { - TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage()); - } else { + TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage()); + } else { TBase::Reply(status, hasIssues); - } + } }); } @@ -296,7 +296,7 @@ namespace NYql::NDqs { }); } - private: + private: THolder<Yql::DqsProto::ExecuteGraphRequest> ModifiedRequest; void DoPassAway() override { @@ -319,9 +319,9 @@ namespace NYql::NDqs { return result; } - void Bootstrap() override { - YQL_LOG(DEBUG) << "TServiceProxyActor::OnExecuteGraph"; - + void Bootstrap() override { + YQL_LOG(DEBUG) << "TServiceProxyActor::OnExecuteGraph"; + SendEvent(NYql::NDqProto::EGraphExecutionEventType::START, SerializeGraphDescriptor(), [this](const TEvGraphExecutionEvent::TPtr& ev) { if (ev->Get()->Record.GetErrorMessage()) { TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage()); @@ -364,21 +364,21 @@ namespace NYql::NDqs { MergeTaskMetas(params); auto executerId = RegisterChild(NDq::MakeDqExecuter(MakeWorkerManagerActorID(SelfId().NodeId()), SelfId(), TraceId, Username, Settings, Counters, RequestStartTime)); - - TVector<TString> columns; - columns.reserve(Request->GetColumns().size()); - for (const auto& column : Request->GetColumns()) { - columns.push_back(column); - } - for (const auto& task : Request->GetTask()) { - Yql::DqsProto::TTaskMeta taskMeta; - task.GetMeta().UnpackTo(&taskMeta); - Task2Stage[task.GetId()] = taskMeta.GetStageId(); - } - THashMap<TString, TString> secureParams; - for (const auto& x : Request->GetSecureParams()) { - secureParams[x.first] = x.second; - } + + TVector<TString> columns; + columns.reserve(Request->GetColumns().size()); + for (const auto& column : Request->GetColumns()) { + columns.push_back(column); + } + for (const auto& task : Request->GetTask()) { + Yql::DqsProto::TTaskMeta taskMeta; + task.GetMeta().UnpackTo(&taskMeta); + Task2Stage[task.GetId()] = taskMeta.GetStageId(); + } + THashMap<TString, TString> secureParams; + for (const auto& x : Request->GetSecureParams()) { + secureParams[x.first] = x.second; + } auto resultId = RegisterChild(NExecutionHelpers::MakeResultAggregator( columns, executerId, @@ -391,10 +391,10 @@ namespace NYql::NDqs { auto controlId = Settings->EnableComputeActor.Get().GetOrElse(false) == false ? resultId : RegisterChild(NYql::MakeTaskController(TraceId, executerId, resultId, Settings, NYq::NCommon::TServiceCounters(Counters, nullptr, "")).Release()); Send(executerId, MakeHolder<TEvGraphRequest>( - *Request, + *Request, controlId, - resultId)); - } + resultId)); + } template <class TPayload, class TCallback> void SendEvent(NYql::NDqProto::EGraphExecutionEventType eventType, const TPayload& payload, TCallback callback) { @@ -412,8 +412,8 @@ namespace NYql::NDqs { } NActors::TActorId GraphExecutionEventsActorId; - }; - + }; + TString GetVersionString() { TStringBuilder sb; sb << GetProgramSvnVersion() << "\n"; @@ -434,50 +434,50 @@ namespace NYql::NDqs { : ActorSystem(system) , Counters(std::move(counters)) , DqTaskPreprocessorFactories(dqTaskPreprocessorFactories) - , Promise(NewPromise<void>()) - , RunningRequests(0) - , Stopping(false) - , Sessions(&system, Counters->GetSubgroup("component", "grpc")->GetCounter("Sessions")) - { } - -#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ - do { \ - MakeIntrusive<NGrpc::TGRpcRequest<Yql::DqsProto::IN, Yql::DqsProto::OUT, TDqsGrpcService>>( \ - this, \ - &Service_, \ - CQ, \ - [this](NGrpc::IRequestContextBase* ctx) { ACTION }, \ - &Yql::DqsProto::DqService::AsyncService::Request##NAME, \ - #NAME, \ - logger, \ - BuildCB(Counters, #NAME)) \ - ->Run(); \ + , Promise(NewPromise<void>()) + , RunningRequests(0) + , Stopping(false) + , Sessions(&system, Counters->GetSubgroup("component", "grpc")->GetCounter("Sessions")) + { } + +#define ADD_REQUEST(NAME, IN, OUT, ACTION) \ + do { \ + MakeIntrusive<NGrpc::TGRpcRequest<Yql::DqsProto::IN, Yql::DqsProto::OUT, TDqsGrpcService>>( \ + this, \ + &Service_, \ + CQ, \ + [this](NGrpc::IRequestContextBase* ctx) { ACTION }, \ + &Yql::DqsProto::DqService::AsyncService::Request##NAME, \ + #NAME, \ + logger, \ + BuildCB(Counters, #NAME)) \ + ->Run(); \ } while (0) void TDqsGrpcService::InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) { using namespace google::protobuf; CQ = cq; - + using TDqTaskPreprocessorCollection = std::vector<NYql::IDqTaskPreprocessor::TPtr>; - ADD_REQUEST(ExecuteGraph, ExecuteGraphRequest, ExecuteGraphResponse, { - TGuard<TMutex> lock(Mutex); - if (Stopping) { - ctx->ReplyError(grpc::UNAVAILABLE, "Terminating in progress"); - return; - } - auto* request = dynamic_cast<const Yql::DqsProto::ExecuteGraphRequest*>(ctx->GetRequest()); - auto session = Sessions.GetSession(request->GetSession()); - if (!session) { - TString message = TStringBuilder() - << "Bad session: " - << request->GetSession(); - YQL_LOG(DEBUG) << message; - ctx->ReplyError(grpc::INVALID_ARGUMENT, message); - return; - } - + ADD_REQUEST(ExecuteGraph, ExecuteGraphRequest, ExecuteGraphResponse, { + TGuard<TMutex> lock(Mutex); + if (Stopping) { + ctx->ReplyError(grpc::UNAVAILABLE, "Terminating in progress"); + return; + } + auto* request = dynamic_cast<const Yql::DqsProto::ExecuteGraphRequest*>(ctx->GetRequest()); + auto session = Sessions.GetSession(request->GetSession()); + if (!session) { + TString message = TStringBuilder() + << "Bad session: " + << request->GetSession(); + YQL_LOG(DEBUG) << message; + ctx->ReplyError(grpc::INVALID_ARGUMENT, message); + return; + } + TDqTaskPreprocessorCollection taskPreprocessors; for (const auto& factory: DqTaskPreprocessorFactories) { taskPreprocessors.push_back(factory()); @@ -485,150 +485,150 @@ namespace NYql::NDqs { auto graphExecutionEventsActorId = ActorSystem.Register(NDqs::MakeGraphExecutionEventsActor(request->GetSession(), std::move(taskPreprocessors))); - RunningRequests++; + RunningRequests++; auto actor = MakeHolder<TExecuteGraphProxyActor>(ctx, Counters, request->GetSession(), session->GetUsername(), graphExecutionEventsActorId); - auto future = actor->GetFuture(); - auto actorId = ActorSystem.Register(actor.Release()); - future.Apply([session, actorId, this] (const TFuture<void>&) mutable { - RunningRequests--; - if (Stopping && !RunningRequests) { - Promise.SetValue(); - } - - session->DeleteRequest(actorId); - }); - session->AddRequest(actorId); - }); - + auto future = actor->GetFuture(); + auto actorId = ActorSystem.Register(actor.Release()); + future.Apply([session, actorId, this] (const TFuture<void>&) mutable { + RunningRequests--; + if (Stopping && !RunningRequests) { + Promise.SetValue(); + } + + session->DeleteRequest(actorId); + }); + session->AddRequest(actorId); + }); + ADD_REQUEST(SvnRevision, SvnRevisionRequest, SvnRevisionResponse, { Y_UNUSED(this); Yql::DqsProto::SvnRevisionResponse result; result.SetRevision(GetVersionString()); - ctx->Reply(&result, Ydb::StatusIds::SUCCESS); + ctx->Reply(&result, Ydb::StatusIds::SUCCESS); + }); + + ADD_REQUEST(CloseSession, CloseSessionRequest, CloseSessionResponse, { + Y_UNUSED(this); + auto* request = dynamic_cast<const Yql::DqsProto::CloseSessionRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + Yql::DqsProto::CloseSessionResponse result; + Sessions.CloseSession(request->GetSession()); + ctx->Reply(&result, Ydb::StatusIds::SUCCESS); }); - - ADD_REQUEST(CloseSession, CloseSessionRequest, CloseSessionResponse, { - Y_UNUSED(this); - auto* request = dynamic_cast<const Yql::DqsProto::CloseSessionRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - Yql::DqsProto::CloseSessionResponse result; - Sessions.CloseSession(request->GetSession()); - ctx->Reply(&result, Ydb::StatusIds::SUCCESS); - }); - - ADD_REQUEST(PingSession, PingSessionRequest, PingSessionResponse, { - Y_UNUSED(this); - auto* request = dynamic_cast<const Yql::DqsProto::PingSessionRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - YQL_LOG(DEBUG) << "PingSession " << request->GetSession(); - - Yql::DqsProto::PingSessionResponse result; - auto session = Sessions.GetSession(request->GetSession()); - if (!session) { - TString message = TStringBuilder() - << "Bad session: " - << request->GetSession(); - YQL_LOG(DEBUG) << message; - ctx->ReplyError(grpc::INVALID_ARGUMENT, message); - } else { - ctx->Reply(&result, Ydb::StatusIds::SUCCESS); - } - }); - - ADD_REQUEST(OpenSession, OpenSessionRequest, OpenSessionResponse, { - Y_UNUSED(this); - auto* request = dynamic_cast<const Yql::DqsProto::OpenSessionRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - YQL_LOG(DEBUG) << "OpenSession for " << request->GetSession() << " " << request->GetUsername(); - - Yql::DqsProto::OpenSessionResponse result; - if (Sessions.OpenSession(request->GetSession(), request->GetUsername())) { - ctx->Reply(&result, Ydb::StatusIds::SUCCESS); - } else { - ctx->ReplyError(grpc::INVALID_ARGUMENT, "Session `" + request->GetSession() + "' exists"); - } - }); - - ADD_REQUEST(JobStop, JobStopRequest, JobStopResponse, { - auto* request = dynamic_cast<const Yql::DqsProto::JobStopRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - auto ev = MakeHolder<TEvJobStop>(*request); - - auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::JobStopResponse>(ctx->GetArena()); - ctx->Reply(result, Ydb::StatusIds::SUCCESS); - - ActorSystem.Send(MakeWorkerManagerActorID(ActorSystem.NodeId), ev.Release()); - }); - - ADD_REQUEST(ClusterStatus, ClusterStatusRequest, ClusterStatusResponse, { - auto ev = MakeHolder<TEvClusterStatus>(); - - using ResultEv = TEvClusterStatusResponse; - - TExecutorPoolStats poolStats; - - TExecutorThreadStats stat; - TVector<TExecutorThreadStats> stats; - ActorSystem.GetPoolStats(0, poolStats, stats); - - for (const auto& s : stats) { - stat.Aggregate(s); - } - - YQL_LOG(DEBUG) << "SentEvents: " << stat.SentEvents; - YQL_LOG(DEBUG) << "ReceivedEvents: " << stat.ReceivedEvents; - YQL_LOG(DEBUG) << "NonDeliveredEvents: " << stat.NonDeliveredEvents; - YQL_LOG(DEBUG) << "EmptyMailboxActivation: " << stat.EmptyMailboxActivation; - Sessions.PrintInfo(); - - for (ui32 i = 0; i < stat.ActorsAliveByActivity.size(); i=i+1) { - if (stat.ActorsAliveByActivity[i]) { - YQL_LOG(DEBUG) << "ActorsAliveByActivity[" << i << "]=" << stat.ActorsAliveByActivity[i]; - } - } - - auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>( - [ctx] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable { - auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::ClusterStatusResponse>(ctx->GetArena()); - result->MergeFrom(event->Get()->Record.GetResponse()); - ctx->Reply(result, Ydb::StatusIds::SUCCESS); - }, - [ctx] () mutable { - YQL_LOG(DEBUG) << "ClusterStatus failed"; - ctx->ReplyError(grpc::UNAVAILABLE, "Error"); - }, - TDuration::MilliSeconds(2000)); - + + ADD_REQUEST(PingSession, PingSessionRequest, PingSessionResponse, { + Y_UNUSED(this); + auto* request = dynamic_cast<const Yql::DqsProto::PingSessionRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + YQL_LOG(DEBUG) << "PingSession " << request->GetSession(); + + Yql::DqsProto::PingSessionResponse result; + auto session = Sessions.GetSession(request->GetSession()); + if (!session) { + TString message = TStringBuilder() + << "Bad session: " + << request->GetSession(); + YQL_LOG(DEBUG) << message; + ctx->ReplyError(grpc::INVALID_ARGUMENT, message); + } else { + ctx->Reply(&result, Ydb::StatusIds::SUCCESS); + } + }); + + ADD_REQUEST(OpenSession, OpenSessionRequest, OpenSessionResponse, { + Y_UNUSED(this); + auto* request = dynamic_cast<const Yql::DqsProto::OpenSessionRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + YQL_LOG(DEBUG) << "OpenSession for " << request->GetSession() << " " << request->GetUsername(); + + Yql::DqsProto::OpenSessionResponse result; + if (Sessions.OpenSession(request->GetSession(), request->GetUsername())) { + ctx->Reply(&result, Ydb::StatusIds::SUCCESS); + } else { + ctx->ReplyError(grpc::INVALID_ARGUMENT, "Session `" + request->GetSession() + "' exists"); + } + }); + + ADD_REQUEST(JobStop, JobStopRequest, JobStopResponse, { + auto* request = dynamic_cast<const Yql::DqsProto::JobStopRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + auto ev = MakeHolder<TEvJobStop>(*request); + + auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::JobStopResponse>(ctx->GetArena()); + ctx->Reply(result, Ydb::StatusIds::SUCCESS); + + ActorSystem.Send(MakeWorkerManagerActorID(ActorSystem.NodeId), ev.Release()); + }); + + ADD_REQUEST(ClusterStatus, ClusterStatusRequest, ClusterStatusResponse, { + auto ev = MakeHolder<TEvClusterStatus>(); + + using ResultEv = TEvClusterStatusResponse; + + TExecutorPoolStats poolStats; + + TExecutorThreadStats stat; + TVector<TExecutorThreadStats> stats; + ActorSystem.GetPoolStats(0, poolStats, stats); + + for (const auto& s : stats) { + stat.Aggregate(s); + } + + YQL_LOG(DEBUG) << "SentEvents: " << stat.SentEvents; + YQL_LOG(DEBUG) << "ReceivedEvents: " << stat.ReceivedEvents; + YQL_LOG(DEBUG) << "NonDeliveredEvents: " << stat.NonDeliveredEvents; + YQL_LOG(DEBUG) << "EmptyMailboxActivation: " << stat.EmptyMailboxActivation; + Sessions.PrintInfo(); + + for (ui32 i = 0; i < stat.ActorsAliveByActivity.size(); i=i+1) { + if (stat.ActorsAliveByActivity[i]) { + YQL_LOG(DEBUG) << "ActorsAliveByActivity[" << i << "]=" << stat.ActorsAliveByActivity[i]; + } + } + + auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>( + [ctx] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable { + auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::ClusterStatusResponse>(ctx->GetArena()); + result->MergeFrom(event->Get()->Record.GetResponse()); + ctx->Reply(result, Ydb::StatusIds::SUCCESS); + }, + [ctx] () mutable { + YQL_LOG(DEBUG) << "ClusterStatus failed"; + ctx->ReplyError(grpc::UNAVAILABLE, "Error"); + }, + TDuration::MilliSeconds(2000)); + + TActorId callbackId = ActorSystem.Register(callback.Release()); + + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); + }); + + ADD_REQUEST(OperationStop, OperationStopRequest, OperationStopResponse, { + auto* request = dynamic_cast<const Yql::DqsProto::OperationStopRequest*>(ctx->GetRequest()); + auto ev = MakeHolder<TEvOperationStop>(*request); + + auto callback = MakeHolder<TRichActorFutureCallback<TEvOperationStopResponse>>( + [ctx] (TAutoPtr<TEventHandle<TEvOperationStopResponse>>& event) mutable { + Y_UNUSED(event); + auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::OperationStopResponse>(ctx->GetArena()); + ctx->Reply(result, Ydb::StatusIds::SUCCESS); + }, + [ctx] () mutable { + YQL_LOG(DEBUG) << "OperationStopResponse failed"; + ctx->ReplyError(grpc::UNAVAILABLE, "Error"); + }, + TDuration::MilliSeconds(2000)); + TActorId callbackId = ActorSystem.Register(callback.Release()); - - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); - }); - - ADD_REQUEST(OperationStop, OperationStopRequest, OperationStopResponse, { - auto* request = dynamic_cast<const Yql::DqsProto::OperationStopRequest*>(ctx->GetRequest()); - auto ev = MakeHolder<TEvOperationStop>(*request); - - auto callback = MakeHolder<TRichActorFutureCallback<TEvOperationStopResponse>>( - [ctx] (TAutoPtr<TEventHandle<TEvOperationStopResponse>>& event) mutable { - Y_UNUSED(event); - auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::OperationStopResponse>(ctx->GetArena()); - ctx->Reply(result, Ydb::StatusIds::SUCCESS); - }, - [ctx] () mutable { - YQL_LOG(DEBUG) << "OperationStopResponse failed"; - ctx->ReplyError(grpc::UNAVAILABLE, "Error"); - }, - TDuration::MilliSeconds(2000)); - - TActorId callbackId = ActorSystem.Register(callback.Release()); - - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); - }); - + + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); + }); + ADD_REQUEST(QueryStatus, QueryStatusRequest, QueryStatusResponse, { auto* request = dynamic_cast<const Yql::DqsProto::QueryStatusRequest*>(ctx->GetRequest()); @@ -648,42 +648,42 @@ namespace NYql::NDqs { TActorId callbackId = ActorSystem.Register(callback.Release()); - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); }); - ADD_REQUEST(RegisterNode, RegisterNodeRequest, RegisterNodeResponse, { - auto* request = dynamic_cast<const Yql::DqsProto::RegisterNodeRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - if (!request->GetPort() - || request->GetRole().empty() - || request->GetAddress().empty() - || request->GetRevision().empty()) - { - ctx->ReplyError(grpc::INVALID_ARGUMENT, "Invalid argument"); - return; - } - - auto ev = MakeHolder<TEvRegisterNode>(*request); - - using ResultEv = TEvRegisterNodeResponse; - - auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>( - [ctx] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable { - auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::RegisterNodeResponse>(ctx->GetArena()); - result->MergeFrom(event->Get()->Record.GetResponse()); - ctx->Reply(result, Ydb::StatusIds::SUCCESS); - }, - [ctx] () mutable { - YQL_LOG(DEBUG) << "RegisterNode failed"; - ctx->ReplyError(grpc::UNAVAILABLE, "Error"); - }, - TDuration::MilliSeconds(5000)); - + ADD_REQUEST(RegisterNode, RegisterNodeRequest, RegisterNodeResponse, { + auto* request = dynamic_cast<const Yql::DqsProto::RegisterNodeRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + if (!request->GetPort() + || request->GetRole().empty() + || request->GetAddress().empty() + || request->GetRevision().empty()) + { + ctx->ReplyError(grpc::INVALID_ARGUMENT, "Invalid argument"); + return; + } + + auto ev = MakeHolder<TEvRegisterNode>(*request); + + using ResultEv = TEvRegisterNodeResponse; + + auto callback = MakeHolder<TRichActorFutureCallback<ResultEv>>( + [ctx] (TAutoPtr<TEventHandle<ResultEv>>& event) mutable { + auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::RegisterNodeResponse>(ctx->GetArena()); + result->MergeFrom(event->Get()->Record.GetResponse()); + ctx->Reply(result, Ydb::StatusIds::SUCCESS); + }, + [ctx] () mutable { + YQL_LOG(DEBUG) << "RegisterNode failed"; + ctx->ReplyError(grpc::UNAVAILABLE, "Error"); + }, + TDuration::MilliSeconds(5000)); + TActorId callbackId = ActorSystem.Register(callback.Release()); - - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); - }); + + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release(), IEventHandle::FlagTrackDelivery)); + }); ADD_REQUEST(GetMaster, GetMasterRequest, GetMasterResponse, { auto* request = dynamic_cast<const Yql::DqsProto::GetMasterRequest*>(ctx->GetRequest()); @@ -700,7 +700,7 @@ namespace NYql::NDqs { TActorId callbackId = ActorSystem.Register(callback.Release()); - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, requestEvent.Release())); + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, requestEvent.Release())); }); ADD_REQUEST(ConfigureFailureInjector, ConfigureFailureInjectorRequest, ConfigureFailureInjectorResponse,{ @@ -718,7 +718,7 @@ namespace NYql::NDqs { TActorId callbackId = ActorSystem.Register(callback.Release()); - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, requestEvent.Release())); + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, requestEvent.Release())); }); ADD_REQUEST(IsReady, IsReadyRequest, IsReadyResponse, { @@ -743,60 +743,60 @@ namespace NYql::NDqs { ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(ActorSystem.NodeId), callbackId, ev.Release())); }); - - ADD_REQUEST(Routes, RoutesRequest, RoutesResponse, { - auto* request = dynamic_cast<const Yql::DqsProto::RoutesRequest*>(ctx->GetRequest()); - Y_VERIFY(!!request); - - auto ev = MakeHolder<TEvRoutesRequest>(); - - auto callback = MakeHolder<TRichActorFutureCallback<TEvRoutesResponse>>( - [ctx] (TAutoPtr<TEventHandle<TEvRoutesResponse>>& event) mutable { - Yql::DqsProto::RoutesResponse result; - result.MergeFrom(event->Get()->Record.GetResponse()); - ctx->Reply(&result, Ydb::StatusIds::SUCCESS); - }, - [ctx] () mutable { - YQL_LOG(DEBUG) << "Routes failed"; - ctx->ReplyError(grpc::UNAVAILABLE, "Error"); - }, - TDuration::MilliSeconds(5000)); - - TActorId callbackId = ActorSystem.Register(callback.Release()); - - ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(request->GetNodeId()), callbackId, ev.Release())); - }); - -/* 1. move grpc to providers/dq, 2. move benchmark to providers/dq 3. uncomment - ADD_REQUEST(Benchmark, BenchmarkRequest, BenchmarkResponse, { - auto* req = dynamic_cast<const Yql::DqsProto::BenchmarkRequest*>(ctx->GetRequest()); - Y_VERIFY(!!req); - - TWorkerManagerBenchmarkOptions options; - if (req->GetWorkerCount()) { - options.WorkerCount = req->GetWorkerCount(); - } - if (req->GetInflight()) { - options.Inflight = req->GetInflight(); - } - if (req->GetTotalRequests()) { - options.TotalRequests = req->GetTotalRequests(); - } - if (req->GetMaxRunTimeMs()) { - options.MaxRunTimeMs = TDuration::MilliSeconds(req->GetMaxRunTimeMs()); - } - - auto benchmarkId = ActorSystem.Register( - CreateWorkerManagerBenchmark( - MakeWorkerManagerActorID(ActorSystem.NodeId), options - )); - - ActorSystem.Send(benchmarkId, new TEvents::TEvBootstrap); - - auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::BenchmarkResponse>(ctx->GetArena()); - ctx->Reply(result, Ydb::StatusIds::SUCCESS); - }); -*/ + + ADD_REQUEST(Routes, RoutesRequest, RoutesResponse, { + auto* request = dynamic_cast<const Yql::DqsProto::RoutesRequest*>(ctx->GetRequest()); + Y_VERIFY(!!request); + + auto ev = MakeHolder<TEvRoutesRequest>(); + + auto callback = MakeHolder<TRichActorFutureCallback<TEvRoutesResponse>>( + [ctx] (TAutoPtr<TEventHandle<TEvRoutesResponse>>& event) mutable { + Yql::DqsProto::RoutesResponse result; + result.MergeFrom(event->Get()->Record.GetResponse()); + ctx->Reply(&result, Ydb::StatusIds::SUCCESS); + }, + [ctx] () mutable { + YQL_LOG(DEBUG) << "Routes failed"; + ctx->ReplyError(grpc::UNAVAILABLE, "Error"); + }, + TDuration::MilliSeconds(5000)); + + TActorId callbackId = ActorSystem.Register(callback.Release()); + + ActorSystem.Send(new IEventHandle(MakeWorkerManagerActorID(request->GetNodeId()), callbackId, ev.Release())); + }); + +/* 1. move grpc to providers/dq, 2. move benchmark to providers/dq 3. uncomment + ADD_REQUEST(Benchmark, BenchmarkRequest, BenchmarkResponse, { + auto* req = dynamic_cast<const Yql::DqsProto::BenchmarkRequest*>(ctx->GetRequest()); + Y_VERIFY(!!req); + + TWorkerManagerBenchmarkOptions options; + if (req->GetWorkerCount()) { + options.WorkerCount = req->GetWorkerCount(); + } + if (req->GetInflight()) { + options.Inflight = req->GetInflight(); + } + if (req->GetTotalRequests()) { + options.TotalRequests = req->GetTotalRequests(); + } + if (req->GetMaxRunTimeMs()) { + options.MaxRunTimeMs = TDuration::MilliSeconds(req->GetMaxRunTimeMs()); + } + + auto benchmarkId = ActorSystem.Register( + CreateWorkerManagerBenchmark( + MakeWorkerManagerActorID(ActorSystem.NodeId), options + )); + + ActorSystem.Send(benchmarkId, new TEvents::TEvBootstrap); + + auto* result = google::protobuf::Arena::CreateMessage<Yql::DqsProto::BenchmarkResponse>(ctx->GetArena()); + ctx->Reply(result, Ydb::StatusIds::SUCCESS); + }); +*/ } void TDqsGrpcService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) { @@ -811,14 +811,14 @@ namespace NYql::NDqs { Limiter->Dec(); Y_ASSERT(Limiter->GetCurrentInFlight() >= 0); } - - TFuture<void> TDqsGrpcService::Stop() { - TGuard<TMutex> lock(Mutex); - Stopping = true; - auto future = Promise.GetFuture(); - if (RunningRequests == 0) { - Promise.SetValue(); - } - return future; - } + + TFuture<void> TDqsGrpcService::Stop() { + TGuard<TMutex> lock(Mutex); + Stopping = true; + auto future = Promise.GetFuture(); + if (RunningRequests == 0) { + Promise.SetValue(); + } + return future; + } } diff --git a/ydb/library/yql/providers/dq/service/grpc_service.h b/ydb/library/yql/providers/dq/service/grpc_service.h index fb29054cc5..fa2a835c8c 100644 --- a/ydb/library/yql/providers/dq/service/grpc_service.h +++ b/ydb/library/yql/providers/dq/service/grpc_service.h @@ -6,7 +6,7 @@ #include <ydb/library/yql/providers/dq/api/protos/service.pb.h> #include <ydb/library/yql/minikql/mkql_function_registry.h> - + #include <library/cpp/grpc/server/grpc_request.h> #include <library/cpp/grpc/server/grpc_server.h> @@ -16,14 +16,14 @@ #include <library/cpp/monlib/dynamic_counters/counters.h> #include <library/cpp/threading/future/future.h> -#include "grpc_session.h" - +#include "grpc_session.h" + namespace NYql::NDqs { class TDatabaseManager; class TDqsGrpcService: public NGrpc::TGrpcServiceBase<Yql::DqsProto::DqService> { public: - TDqsGrpcService(NActors::TActorSystem& system, + TDqsGrpcService(NActors::TActorSystem& system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories); @@ -33,8 +33,8 @@ namespace NYql::NDqs { bool IncRequest(); void DecRequest(); - NThreading::TFuture<void> Stop(); - + NThreading::TFuture<void> Stop(); + private: NActors::TActorSystem& ActorSystem; grpc::ServerCompletionQueue* CQ = nullptr; @@ -42,11 +42,11 @@ namespace NYql::NDqs { TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; TDqTaskPreprocessorFactoryCollection DqTaskPreprocessorFactories; - TMutex Mutex; + TMutex Mutex; NThreading::TPromise<void> Promise; - std::atomic<ui64> RunningRequests; - std::atomic<bool> Stopping; - - TSessionStorage Sessions; + std::atomic<ui64> RunningRequests; + std::atomic<bool> Stopping; + + TSessionStorage Sessions; }; } diff --git a/ydb/library/yql/providers/dq/service/grpc_session.cpp b/ydb/library/yql/providers/dq/service/grpc_session.cpp index 7f3acce885..38e95f6668 100644 --- a/ydb/library/yql/providers/dq/service/grpc_session.cpp +++ b/ydb/library/yql/providers/dq/service/grpc_session.cpp @@ -1,106 +1,106 @@ -#include "grpc_session.h" - +#include "grpc_session.h" + #include <ydb/library/yql/utils/log/log.h> - -namespace NYql::NDqs { - -TSession::~TSession() { - TGuard<TMutex> lock(Mutex); - for (auto actorId : Requests) { - ActorSystem->Send(actorId, new NActors::TEvents::TEvPoison()); - } -} - -void TSession::DeleteRequest(const NActors::TActorId& actorId) -{ - TGuard<TMutex> lock(Mutex); - Requests.erase(actorId); -} - -void TSession::AddRequest(const NActors::TActorId& actorId) -{ - TGuard<TMutex> lock(Mutex); - Requests.insert(actorId); -} - -TSessionStorage::TSessionStorage( - NActors::TActorSystem* actorSystem, - const NMonitoring::TDynamicCounters::TCounterPtr& sessionsCounter) - : ActorSystem(actorSystem) - , SessionsCounter(sessionsCounter) -{ } - -void TSessionStorage::CloseSession(const TString& sessionId) -{ - TGuard<TMutex> lock(SessionMutex); - auto it = Sessions.find(sessionId); - if (it == Sessions.end()) { - return; - } - SessionsByLastUpdate.erase(it->second.Iterator); - Sessions.erase(it); - *SessionsCounter = Sessions.size(); -} - -std::shared_ptr<TSession> TSessionStorage::GetSession(const TString& sessionId) -{ - Clean(TInstant::Now() - TDuration::Minutes(10)); - - TGuard<TMutex> lock(SessionMutex); - auto it = Sessions.find(sessionId); - if (it == Sessions.end()) { - return std::shared_ptr<TSession>(); - } else { - SessionsByLastUpdate.erase(it->second.Iterator); - SessionsByLastUpdate.push_back({TInstant::Now(), sessionId}); - it->second.Iterator = SessionsByLastUpdate.end(); - it->second.Iterator--; - return it->second.Session; - } -} - -bool TSessionStorage::OpenSession(const TString& sessionId, const TString& username) -{ - TGuard<TMutex> lock(SessionMutex); - if (Sessions.contains(sessionId)) { - return false; - } - - SessionsByLastUpdate.push_back({TInstant::Now(), sessionId}); - auto it = SessionsByLastUpdate.end(); --it; - - Sessions[sessionId] = TSessionAndIterator { - std::make_shared<TSession>(username, ActorSystem), - it - }; - - *SessionsCounter = Sessions.size(); - - return true; -} - -void TSessionStorage::Clean(TInstant before) { - TGuard<TMutex> lock(SessionMutex); - for (TSessionsByLastUpdate::iterator it = SessionsByLastUpdate.begin(); - it != SessionsByLastUpdate.end(); ) - { - if (it->LastUpdate < before) { - YQL_LOG(INFO) << "Drop session by timeout " << it->SessionId; - Sessions.erase(it->SessionId); - it = SessionsByLastUpdate.erase(it); - } else { - break; - } - } - - *SessionsCounter = Sessions.size(); -} - -void TSessionStorage::PrintInfo() const { - YQL_LOG(INFO) << "SessionsByLastUpdate: " << SessionsByLastUpdate.size(); - YQL_LOG(DEBUG) << "Sessions: " << Sessions.size(); - ui64 currenSessionsCounter = *SessionsCounter; - YQL_LOG(DEBUG) << "SessionsCounter: " << currenSessionsCounter; -} - -} // namespace NYql::NDqs + +namespace NYql::NDqs { + +TSession::~TSession() { + TGuard<TMutex> lock(Mutex); + for (auto actorId : Requests) { + ActorSystem->Send(actorId, new NActors::TEvents::TEvPoison()); + } +} + +void TSession::DeleteRequest(const NActors::TActorId& actorId) +{ + TGuard<TMutex> lock(Mutex); + Requests.erase(actorId); +} + +void TSession::AddRequest(const NActors::TActorId& actorId) +{ + TGuard<TMutex> lock(Mutex); + Requests.insert(actorId); +} + +TSessionStorage::TSessionStorage( + NActors::TActorSystem* actorSystem, + const NMonitoring::TDynamicCounters::TCounterPtr& sessionsCounter) + : ActorSystem(actorSystem) + , SessionsCounter(sessionsCounter) +{ } + +void TSessionStorage::CloseSession(const TString& sessionId) +{ + TGuard<TMutex> lock(SessionMutex); + auto it = Sessions.find(sessionId); + if (it == Sessions.end()) { + return; + } + SessionsByLastUpdate.erase(it->second.Iterator); + Sessions.erase(it); + *SessionsCounter = Sessions.size(); +} + +std::shared_ptr<TSession> TSessionStorage::GetSession(const TString& sessionId) +{ + Clean(TInstant::Now() - TDuration::Minutes(10)); + + TGuard<TMutex> lock(SessionMutex); + auto it = Sessions.find(sessionId); + if (it == Sessions.end()) { + return std::shared_ptr<TSession>(); + } else { + SessionsByLastUpdate.erase(it->second.Iterator); + SessionsByLastUpdate.push_back({TInstant::Now(), sessionId}); + it->second.Iterator = SessionsByLastUpdate.end(); + it->second.Iterator--; + return it->second.Session; + } +} + +bool TSessionStorage::OpenSession(const TString& sessionId, const TString& username) +{ + TGuard<TMutex> lock(SessionMutex); + if (Sessions.contains(sessionId)) { + return false; + } + + SessionsByLastUpdate.push_back({TInstant::Now(), sessionId}); + auto it = SessionsByLastUpdate.end(); --it; + + Sessions[sessionId] = TSessionAndIterator { + std::make_shared<TSession>(username, ActorSystem), + it + }; + + *SessionsCounter = Sessions.size(); + + return true; +} + +void TSessionStorage::Clean(TInstant before) { + TGuard<TMutex> lock(SessionMutex); + for (TSessionsByLastUpdate::iterator it = SessionsByLastUpdate.begin(); + it != SessionsByLastUpdate.end(); ) + { + if (it->LastUpdate < before) { + YQL_LOG(INFO) << "Drop session by timeout " << it->SessionId; + Sessions.erase(it->SessionId); + it = SessionsByLastUpdate.erase(it); + } else { + break; + } + } + + *SessionsCounter = Sessions.size(); +} + +void TSessionStorage::PrintInfo() const { + YQL_LOG(INFO) << "SessionsByLastUpdate: " << SessionsByLastUpdate.size(); + YQL_LOG(DEBUG) << "Sessions: " << Sessions.size(); + ui64 currenSessionsCounter = *SessionsCounter; + YQL_LOG(DEBUG) << "SessionsCounter: " << currenSessionsCounter; +} + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/service/grpc_session.h b/ydb/library/yql/providers/dq/service/grpc_session.h index fa25b4982a..56e592a757 100644 --- a/ydb/library/yql/providers/dq/service/grpc_session.h +++ b/ydb/library/yql/providers/dq/service/grpc_session.h @@ -1,57 +1,57 @@ -#pragma once -#include <library/cpp/actors/core/actorsystem.h> - -namespace NYql::NDqs { - -class TSession { -public: - TSession(const TString& username, NActors::TActorSystem* actorSystem) - : Username(username) - , ActorSystem(actorSystem) - { } - - ~TSession(); - - const TString& GetUsername() const { - return Username; - } - - void DeleteRequest(const NActors::TActorId& id); - void AddRequest(const NActors::TActorId& id); - -private: - TMutex Mutex; - const TString Username; - NActors::TActorSystem* ActorSystem; - THashSet<NActors::TActorId> Requests; -}; - -class TSessionStorage { -public: - TSessionStorage( - NActors::TActorSystem* actorSystem, - const NMonitoring::TDynamicCounters::TCounterPtr& sessionsCounter); - void CloseSession(const TString& sessionId); - std::shared_ptr<TSession> GetSession(const TString& sessionId); - bool OpenSession(const TString& sessionId, const TString& username); - void Clean(TInstant before); - void PrintInfo() const; - -private: - NActors::TActorSystem* ActorSystem; - NMonitoring::TDynamicCounters::TCounterPtr SessionsCounter; - TMutex SessionMutex; - struct TTimeAndSessionId { - TInstant LastUpdate; - TString SessionId; - }; - using TSessionsByLastUpdate = TList<TTimeAndSessionId>; - TSessionsByLastUpdate SessionsByLastUpdate; - struct TSessionAndIterator { - std::shared_ptr<TSession> Session; - TSessionsByLastUpdate::iterator Iterator; - }; - THashMap<TString, TSessionAndIterator> Sessions; -}; - -} // namespace NYql::NDqs +#pragma once +#include <library/cpp/actors/core/actorsystem.h> + +namespace NYql::NDqs { + +class TSession { +public: + TSession(const TString& username, NActors::TActorSystem* actorSystem) + : Username(username) + , ActorSystem(actorSystem) + { } + + ~TSession(); + + const TString& GetUsername() const { + return Username; + } + + void DeleteRequest(const NActors::TActorId& id); + void AddRequest(const NActors::TActorId& id); + +private: + TMutex Mutex; + const TString Username; + NActors::TActorSystem* ActorSystem; + THashSet<NActors::TActorId> Requests; +}; + +class TSessionStorage { +public: + TSessionStorage( + NActors::TActorSystem* actorSystem, + const NMonitoring::TDynamicCounters::TCounterPtr& sessionsCounter); + void CloseSession(const TString& sessionId); + std::shared_ptr<TSession> GetSession(const TString& sessionId); + bool OpenSession(const TString& sessionId, const TString& username); + void Clean(TInstant before); + void PrintInfo() const; + +private: + NActors::TActorSystem* ActorSystem; + NMonitoring::TDynamicCounters::TCounterPtr SessionsCounter; + TMutex SessionMutex; + struct TTimeAndSessionId { + TInstant LastUpdate; + TString SessionId; + }; + using TSessionsByLastUpdate = TList<TTimeAndSessionId>; + TSessionsByLastUpdate SessionsByLastUpdate; + struct TSessionAndIterator { + std::shared_ptr<TSession> Session; + TSessionsByLastUpdate::iterator Iterator; + }; + THashMap<TString, TSessionAndIterator> Sessions; +}; + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/service/interconnect_helpers.cpp b/ydb/library/yql/providers/dq/service/interconnect_helpers.cpp index b873c85a77..bef368a961 100644 --- a/ydb/library/yql/providers/dq/service/interconnect_helpers.cpp +++ b/ydb/library/yql/providers/dq/service/interconnect_helpers.cpp @@ -1,19 +1,19 @@ #include "interconnect_helpers.h" -#include "service_node.h" +#include "service_node.h" #include "grpc_service.h" -#include <library/cpp/actors/helpers/selfping_actor.h> - +#include <library/cpp/actors/helpers/selfping_actor.h> + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/backtrace/backtrace.h> #include <ydb/library/yql/utils/yql_panic.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> - + #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/scheduler_basic.h> -#include <library/cpp/actors/core/scheduler_actor.h> +#include <library/cpp/actors/core/scheduler_actor.h> #include <library/cpp/actors/dnsresolver/dnsresolver.h> #include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/actors/interconnect/interconnect_common.h> @@ -23,281 +23,281 @@ #include <library/cpp/yson/node/node_io.h> #include <util/stream/file.h> -#include <util/system/env.h> +#include <util/system/env.h> namespace NYql::NDqs { using namespace NActors; using namespace NActors::NDnsResolver; using namespace NGrpc; - class TYqlLogBackend: public TLogBackend { - void WriteData(const TLogRecord& rec) override { - TString message(rec.Data, rec.Len); - if (message.find("ICP01 ready to work") != TString::npos) { - return; - } - YQL_LOG(DEBUG) << message; - } - - void ReopenLog() override { } - }; - - static void InitSelfPingActor(NActors::TActorSystemSetup* setup, NMonitoring::TDynamicCounterPtr rootCounters) - { - const TDuration selfPingInterval = TDuration::MilliSeconds(10); - - const auto counters = rootCounters->GetSubgroup("counters", "utils"); - - for (size_t poolId = 0; poolId < setup->GetExecutorsCount(); ++poolId) { - const auto& poolName = setup->GetPoolName(poolId); - auto poolGroup = counters->GetSubgroup("execpool", poolName); - auto counter = poolGroup->GetCounter("SelfPingMaxUs", false); - auto cpuTimeCounter = poolGroup->GetCounter("CpuMatBenchNs", false); - IActor* selfPingActor = CreateSelfPingActor(selfPingInterval, counter, cpuTimeCounter); - setup->LocalServices.push_back( - std::make_pair(TActorId(), - TActorSetupCmd(selfPingActor, - TMailboxType::HTSwap, - poolId))); - } - } - - std::tuple<THolder<NActors::TActorSystemSetup>, TIntrusivePtr<NActors::NLog::TSettings>> BuildActorSetup( - ui32 nodeId, - TString interconnectAddress, - ui16 port, - SOCKET socket, - TVector<ui32> threads, - NMonitoring::TDynamicCounterPtr counters, - const TNameserverFactory& nameserverFactory, - const NYql::NProto::TDqConfig::TICSettings& icSettings) - { - auto setup = MakeHolder<TActorSystemSetup>(); - - setup->NodeId = nodeId; - - const int maxActivityType = NActors::GetActivityTypeCount(); - if (threads.empty()) { - threads = {icSettings.GetThreads()}; - } - - setup->ExecutorsCount = threads.size(); + class TYqlLogBackend: public TLogBackend { + void WriteData(const TLogRecord& rec) override { + TString message(rec.Data, rec.Len); + if (message.find("ICP01 ready to work") != TString::npos) { + return; + } + YQL_LOG(DEBUG) << message; + } + + void ReopenLog() override { } + }; + + static void InitSelfPingActor(NActors::TActorSystemSetup* setup, NMonitoring::TDynamicCounterPtr rootCounters) + { + const TDuration selfPingInterval = TDuration::MilliSeconds(10); + + const auto counters = rootCounters->GetSubgroup("counters", "utils"); + + for (size_t poolId = 0; poolId < setup->GetExecutorsCount(); ++poolId) { + const auto& poolName = setup->GetPoolName(poolId); + auto poolGroup = counters->GetSubgroup("execpool", poolName); + auto counter = poolGroup->GetCounter("SelfPingMaxUs", false); + auto cpuTimeCounter = poolGroup->GetCounter("CpuMatBenchNs", false); + IActor* selfPingActor = CreateSelfPingActor(selfPingInterval, counter, cpuTimeCounter); + setup->LocalServices.push_back( + std::make_pair(TActorId(), + TActorSetupCmd(selfPingActor, + TMailboxType::HTSwap, + poolId))); + } + } + + std::tuple<THolder<NActors::TActorSystemSetup>, TIntrusivePtr<NActors::NLog::TSettings>> BuildActorSetup( + ui32 nodeId, + TString interconnectAddress, + ui16 port, + SOCKET socket, + TVector<ui32> threads, + NMonitoring::TDynamicCounterPtr counters, + const TNameserverFactory& nameserverFactory, + const NYql::NProto::TDqConfig::TICSettings& icSettings) + { + auto setup = MakeHolder<TActorSystemSetup>(); + + setup->NodeId = nodeId; + + const int maxActivityType = NActors::GetActivityTypeCount(); + if (threads.empty()) { + threads = {icSettings.GetThreads()}; + } + + setup->ExecutorsCount = threads.size(); setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); - for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { - setup->Executors[i] = new TBasicExecutorPool( - i, - threads[i], - 50, - "pool-"+ToString(i), // poolName - nullptr, // affinity - NActors::TBasicExecutorPool::DEFAULT_TIME_PER_MAILBOX, // timePerMailbox - NActors::TBasicExecutorPool::DEFAULT_EVENTS_PER_MAILBOX, // eventsPermailbox - 0, // realtimePriority - maxActivityType // maxActivityType - ); - } - auto schedulerConfig = TSchedulerConfig(); - schedulerConfig.MonCounters = counters; - -#define SET_VALUE(name) \ - if (icSettings.Has ## name()) { \ - schedulerConfig.name = icSettings.Get ## name (); \ - YQL_LOG(DEBUG) << "Scheduler IC " << #name << " set to " << schedulerConfig.name; \ - } - - SET_VALUE(ResolutionMicroseconds); - SET_VALUE(SpinThreshold); - SET_VALUE(ProgressThreshold); - SET_VALUE(UseSchedulerActor); - SET_VALUE(RelaxedSendPaceEventsPerSecond); - SET_VALUE(RelaxedSendPaceEventsPerCycle); - SET_VALUE(RelaxedSendThresholdEventsPerSecond); - SET_VALUE(RelaxedSendThresholdEventsPerCycle); - -#undef SET_VALUE - - setup->Scheduler = CreateSchedulerThread(schedulerConfig); - setup->MaxActivityType = maxActivityType; - - YQL_LOG(DEBUG) << "Initializing local services"; + for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { + setup->Executors[i] = new TBasicExecutorPool( + i, + threads[i], + 50, + "pool-"+ToString(i), // poolName + nullptr, // affinity + NActors::TBasicExecutorPool::DEFAULT_TIME_PER_MAILBOX, // timePerMailbox + NActors::TBasicExecutorPool::DEFAULT_EVENTS_PER_MAILBOX, // eventsPermailbox + 0, // realtimePriority + maxActivityType // maxActivityType + ); + } + auto schedulerConfig = TSchedulerConfig(); + schedulerConfig.MonCounters = counters; + +#define SET_VALUE(name) \ + if (icSettings.Has ## name()) { \ + schedulerConfig.name = icSettings.Get ## name (); \ + YQL_LOG(DEBUG) << "Scheduler IC " << #name << " set to " << schedulerConfig.name; \ + } + + SET_VALUE(ResolutionMicroseconds); + SET_VALUE(SpinThreshold); + SET_VALUE(ProgressThreshold); + SET_VALUE(UseSchedulerActor); + SET_VALUE(RelaxedSendPaceEventsPerSecond); + SET_VALUE(RelaxedSendPaceEventsPerCycle); + SET_VALUE(RelaxedSendThresholdEventsPerSecond); + SET_VALUE(RelaxedSendThresholdEventsPerCycle); + +#undef SET_VALUE + + setup->Scheduler = CreateSchedulerThread(schedulerConfig); + setup->MaxActivityType = maxActivityType; + + YQL_LOG(DEBUG) << "Initializing local services"; setup->LocalServices.emplace_back(MakePollerActorId(), TActorSetupCmd(CreatePollerActor(), TMailboxType::ReadAsFilled, 0)); - if (IActor* schedulerActor = CreateSchedulerActor(schedulerConfig)) { - TActorId schedulerActorId = MakeSchedulerActorId(); - setup->LocalServices.emplace_back(schedulerActorId, TActorSetupCmd(schedulerActor, TMailboxType::ReadAsFilled, 0)); - } - - NActors::TActorId loggerActorId(nodeId, "logger"); - auto logSettings = MakeIntrusive<NActors::NLog::TSettings>(loggerActorId, - 0, NActors::NLog::PRI_INFO); - static TString defaultComponent = "ActorLib"; - logSettings->Append(0, 1024, [&](NActors::NLog::EComponent) -> const TString & { return defaultComponent; }); - TString wtf = ""; - logSettings->SetLevel(NActors::NLog::PRI_DEBUG, 535 /*NKikimrServices::KQP_COMPUTE*/, wtf); - logSettings->SetLevel(NActors::NLog::PRI_DEBUG, 713 /*NKikimrServices::YQL_PROXY*/, wtf); - NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor( - logSettings, - new TYqlLogBackend, - counters->GetSubgroup("logger", "counters")); - setup->LocalServices.emplace_back(logSettings->LoggerActorId, TActorSetupCmd(loggerActor, TMailboxType::Simple, 0)); - - TIntrusivePtr<TTableNameserverSetup> nameserverTable = new TTableNameserverSetup(); - THashSet<ui32> staticNodeId; - - YQL_LOG(DEBUG) << "Initializing node table"; - nameserverTable->StaticNodeTable[nodeId] = std::make_pair(interconnectAddress, port); - - setup->LocalServices.emplace_back( + if (IActor* schedulerActor = CreateSchedulerActor(schedulerConfig)) { + TActorId schedulerActorId = MakeSchedulerActorId(); + setup->LocalServices.emplace_back(schedulerActorId, TActorSetupCmd(schedulerActor, TMailboxType::ReadAsFilled, 0)); + } + + NActors::TActorId loggerActorId(nodeId, "logger"); + auto logSettings = MakeIntrusive<NActors::NLog::TSettings>(loggerActorId, + 0, NActors::NLog::PRI_INFO); + static TString defaultComponent = "ActorLib"; + logSettings->Append(0, 1024, [&](NActors::NLog::EComponent) -> const TString & { return defaultComponent; }); + TString wtf = ""; + logSettings->SetLevel(NActors::NLog::PRI_DEBUG, 535 /*NKikimrServices::KQP_COMPUTE*/, wtf); + logSettings->SetLevel(NActors::NLog::PRI_DEBUG, 713 /*NKikimrServices::YQL_PROXY*/, wtf); + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor( + logSettings, + new TYqlLogBackend, + counters->GetSubgroup("logger", "counters")); + setup->LocalServices.emplace_back(logSettings->LoggerActorId, TActorSetupCmd(loggerActor, TMailboxType::Simple, 0)); + + TIntrusivePtr<TTableNameserverSetup> nameserverTable = new TTableNameserverSetup(); + THashSet<ui32> staticNodeId; + + YQL_LOG(DEBUG) << "Initializing node table"; + nameserverTable->StaticNodeTable[nodeId] = std::make_pair(interconnectAddress, port); + + setup->LocalServices.emplace_back( MakeDnsResolverActorId(), TActorSetupCmd(CreateOnDemandDnsResolver(), TMailboxType::ReadAsFilled, 0)); setup->LocalServices.emplace_back( - GetNameserviceActorId(), TActorSetupCmd(nameserverFactory(nameserverTable), TMailboxType::ReadAsFilled, 0)); + GetNameserviceActorId(), TActorSetupCmd(nameserverFactory(nameserverTable), TMailboxType::ReadAsFilled, 0)); + - - InitSelfPingActor(setup.Get(), counters); - - TIntrusivePtr<TInterconnectProxyCommon> icCommon = new TInterconnectProxyCommon(); + InitSelfPingActor(setup.Get(), counters); + + TIntrusivePtr<TInterconnectProxyCommon> icCommon = new TInterconnectProxyCommon(); icCommon->NameserviceId = GetNameserviceActorId(); - Y_UNUSED(counters); - //icCommon->MonCounters = counters->GetSubgroup("counters", "interconnect"); - icCommon->MonCounters = MakeIntrusive<NMonitoring::TDynamicCounters>(); - -#define SET_DURATION(name) \ - { \ - icCommon->Settings.name = TDuration::MilliSeconds(icSettings.Get ## name ## Ms()); \ - YQL_LOG(DEBUG) << "IC " << #name << " set to " << icCommon->Settings.name; \ - } - -#define SET_VALUE(name) \ - { \ - icCommon->Settings.name = icSettings.Get ## name(); \ - YQL_LOG(DEBUG) << "IC " << #name << " set to " << icCommon->Settings.name; \ - } - - SET_DURATION(Handshake); - SET_DURATION(DeadPeer); - SET_DURATION(CloseOnIdle); - - SET_VALUE(SendBufferDieLimitInMB); - SET_VALUE(TotalInflightAmountOfData); - SET_VALUE(MergePerPeerCounters); - SET_VALUE(MergePerDataCenterCounters); - SET_VALUE(TCPSocketBufferSize); - - SET_DURATION(PingPeriod); - SET_DURATION(ForceConfirmPeriod); - SET_DURATION(LostConnection); - SET_DURATION(BatchPeriod); - - SET_DURATION(MessagePendingTimeout); - - SET_VALUE(MessagePendingSize); - SET_VALUE(MaxSerializedEventSize); - -#undef SET_DURATION -#undef SET_VALUE - - ui32 maxNodeId = static_cast<ui32>(ENodeIdLimits::MaxWorkerNodeId); - - YQL_LOG(DEBUG) << "Initializing proxy actors"; - setup->Interconnect.ProxyActors.resize(maxNodeId + 1); - for (ui32 id = 1; id <= maxNodeId; ++id) { - if (nodeId != id) { + Y_UNUSED(counters); + //icCommon->MonCounters = counters->GetSubgroup("counters", "interconnect"); + icCommon->MonCounters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + +#define SET_DURATION(name) \ + { \ + icCommon->Settings.name = TDuration::MilliSeconds(icSettings.Get ## name ## Ms()); \ + YQL_LOG(DEBUG) << "IC " << #name << " set to " << icCommon->Settings.name; \ + } + +#define SET_VALUE(name) \ + { \ + icCommon->Settings.name = icSettings.Get ## name(); \ + YQL_LOG(DEBUG) << "IC " << #name << " set to " << icCommon->Settings.name; \ + } + + SET_DURATION(Handshake); + SET_DURATION(DeadPeer); + SET_DURATION(CloseOnIdle); + + SET_VALUE(SendBufferDieLimitInMB); + SET_VALUE(TotalInflightAmountOfData); + SET_VALUE(MergePerPeerCounters); + SET_VALUE(MergePerDataCenterCounters); + SET_VALUE(TCPSocketBufferSize); + + SET_DURATION(PingPeriod); + SET_DURATION(ForceConfirmPeriod); + SET_DURATION(LostConnection); + SET_DURATION(BatchPeriod); + + SET_DURATION(MessagePendingTimeout); + + SET_VALUE(MessagePendingSize); + SET_VALUE(MaxSerializedEventSize); + +#undef SET_DURATION +#undef SET_VALUE + + ui32 maxNodeId = static_cast<ui32>(ENodeIdLimits::MaxWorkerNodeId); + + YQL_LOG(DEBUG) << "Initializing proxy actors"; + setup->Interconnect.ProxyActors.resize(maxNodeId + 1); + for (ui32 id = 1; id <= maxNodeId; ++id) { + if (nodeId != id) { IActor* actor = new TInterconnectProxyTCP(id, icCommon); - setup->Interconnect.ProxyActors[id] = TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0); + setup->Interconnect.ProxyActors[id] = TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0); } - } - - // start listener - YQL_LOG(DEBUG) << "Start listener"; - { - icCommon->TechnicalSelfHostName = interconnectAddress; - YQL_LOG(INFO) << "Start listener " << interconnectAddress << ":" << port << " socket: " << socket; - IActor* listener; - TMaybe<SOCKET> maybeSocket = socket < 0 - ? Nothing() - : TMaybe<SOCKET>(socket); - - listener = new NActors::TInterconnectListenerTCP(interconnectAddress, port, icCommon, maybeSocket); - - setup->LocalServices.emplace_back( - MakeInterconnectListenerActorId(false), - TActorSetupCmd(listener, TMailboxType::ReadAsFilled, 0)); - } - - YQL_LOG(DEBUG) << "Actor initialization complete"; - + } + + // start listener + YQL_LOG(DEBUG) << "Start listener"; + { + icCommon->TechnicalSelfHostName = interconnectAddress; + YQL_LOG(INFO) << "Start listener " << interconnectAddress << ":" << port << " socket: " << socket; + IActor* listener; + TMaybe<SOCKET> maybeSocket = socket < 0 + ? Nothing() + : TMaybe<SOCKET>(socket); + + listener = new NActors::TInterconnectListenerTCP(interconnectAddress, port, icCommon, maybeSocket); + + setup->LocalServices.emplace_back( + MakeInterconnectListenerActorId(false), + TActorSetupCmd(listener, TMailboxType::ReadAsFilled, 0)); + } + + YQL_LOG(DEBUG) << "Actor initialization complete"; + #ifdef _unix_ - signal(SIGPIPE, SIG_IGN); + signal(SIGPIPE, SIG_IGN); #endif - return std::make_tuple(std::move(setup), logSettings); + return std::make_tuple(std::move(setup), logSettings); } - std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname) { - constexpr auto MaxLocalHostNameLength = 4096; - std::array<char, MaxLocalHostNameLength> buffer; - buffer.fill(0); - TString hostName; - TString localAddress; - - int result = gethostname(buffer.data(), buffer.size() - 1); - if (result != 0) { + std::tuple<TString, TString> GetLocalAddress(const TString* overrideHostname) { + constexpr auto MaxLocalHostNameLength = 4096; + std::array<char, MaxLocalHostNameLength> buffer; + buffer.fill(0); + TString hostName; + TString localAddress; + + int result = gethostname(buffer.data(), buffer.size() - 1); + if (result != 0) { Cerr << "gethostname failed for " << std::string_view{buffer.data(), buffer.size()} << " error " << strerror(errno) << Endl; - return std::make_tuple(hostName, localAddress); - } - - if (overrideHostname) { - memcpy(&buffer[0], overrideHostname->c_str(), Min<int>( - overrideHostname->size()+1, buffer.size()-1 - )); - } - - hostName = &buffer[0]; - - addrinfo request; - memset(&request, 0, sizeof(request)); - request.ai_family = AF_INET6; - request.ai_socktype = SOCK_STREAM; - - addrinfo* response = nullptr; - result = getaddrinfo(buffer.data(), nullptr, &request, &response); - if (result != 0) { + return std::make_tuple(hostName, localAddress); + } + + if (overrideHostname) { + memcpy(&buffer[0], overrideHostname->c_str(), Min<int>( + overrideHostname->size()+1, buffer.size()-1 + )); + } + + hostName = &buffer[0]; + + addrinfo request; + memset(&request, 0, sizeof(request)); + request.ai_family = AF_INET6; + request.ai_socktype = SOCK_STREAM; + + addrinfo* response = nullptr; + result = getaddrinfo(buffer.data(), nullptr, &request, &response); + if (result != 0) { Cerr << "getaddrinfo failed for " << std::string_view{buffer.data(), buffer.size()} << " error " << gai_strerror(result) << Endl; - return std::make_tuple(hostName, localAddress); - } - - std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo); - - if (!response->ai_addr) { - Cerr << "getaddrinfo failed: no ai_addr" << Endl; - return std::make_tuple(hostName, localAddress); - } - - auto* sa = response->ai_addr; - Y_VERIFY(sa->sa_family == AF_INET6); - inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr), - &buffer[0], buffer.size() - 1); - - localAddress = &buffer[0]; - - return std::make_tuple(hostName, localAddress); - } - - std::tuple<TString, TString> GetUserToken(const TMaybe<TString>& maybeUser, const TMaybe<TString>& maybeTokenFile) - { - auto home = GetEnv("HOME"); - auto systemUser = GetEnv("USER"); - - TString userName = maybeUser - ? *maybeUser - : systemUser; - - TString tokenFile = maybeTokenFile - ? *maybeTokenFile - : home + "/.yt/token"; - - TString token = TFileInput(tokenFile).ReadLine(); - - return std::make_tuple(userName, token); - } + return std::make_tuple(hostName, localAddress); + } + + std::unique_ptr<addrinfo, void (*)(addrinfo*)> holder(response, &freeaddrinfo); + + if (!response->ai_addr) { + Cerr << "getaddrinfo failed: no ai_addr" << Endl; + return std::make_tuple(hostName, localAddress); + } + + auto* sa = response->ai_addr; + Y_VERIFY(sa->sa_family == AF_INET6); + inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr), + &buffer[0], buffer.size() - 1); + + localAddress = &buffer[0]; + + return std::make_tuple(hostName, localAddress); + } + + std::tuple<TString, TString> GetUserToken(const TMaybe<TString>& maybeUser, const TMaybe<TString>& maybeTokenFile) + { + auto home = GetEnv("HOME"); + auto systemUser = GetEnv("USER"); + + TString userName = maybeUser + ? *maybeUser + : systemUser; + + TString tokenFile = maybeTokenFile + ? *maybeTokenFile + : home + "/.yt/token"; + + TString token = TFileInput(tokenFile).ReadLine(); + + return std::make_tuple(userName, token); + } } diff --git a/ydb/library/yql/providers/dq/service/interconnect_helpers.h b/ydb/library/yql/providers/dq/service/interconnect_helpers.h index 8808615817..9009c8c179 100644 --- a/ydb/library/yql/providers/dq/service/interconnect_helpers.h +++ b/ydb/library/yql/providers/dq/service/interconnect_helpers.h @@ -2,48 +2,48 @@ #include <library/cpp/actors/core/actorsystem.h> #include <library/cpp/actors/interconnect/poller_tcp.h> -#include <library/cpp/actors/interconnect/interconnect.h> +#include <library/cpp/actors/interconnect/interconnect.h> #include <library/cpp/yson/node/node.h> -#include <ydb/library/yql/providers/dq/config/config.pb.h> - +#include <ydb/library/yql/providers/dq/config/config.pb.h> + namespace NYql::NDqs { - -enum class ENodeIdLimits { - MinServiceNodeId = 1, - MaxServiceNodeId = 512, // excluding - MinWorkerNodeId = 512, - MaxWorkerNodeId = 8192, // excluding -}; - -using TNameserverFactory = std::function<NActors::IActor*(const TIntrusivePtr<NActors::TTableNameserverSetup>& setup)>; - -struct TServiceNodeConfig { - ui32 NodeId; - TString InterconnectAddress; - TString GrpcHostname; - ui16 Port; - ui16 GrpcPort = 8080; - ui16 MbusPort = 0; - SOCKET Socket = -1; - SOCKET GrpcSocket = -1; - NYql::NProto::TDqConfig::TICSettings ICSettings = NYql::NProto::TDqConfig::TICSettings(); - TNameserverFactory NameserverFactory = [](const TIntrusivePtr<NActors::TTableNameserverSetup>& setup) { - return CreateNameserverTable(setup); + +enum class ENodeIdLimits { + MinServiceNodeId = 1, + MaxServiceNodeId = 512, // excluding + MinWorkerNodeId = 512, + MaxWorkerNodeId = 8192, // excluding +}; + +using TNameserverFactory = std::function<NActors::IActor*(const TIntrusivePtr<NActors::TTableNameserverSetup>& setup)>; + +struct TServiceNodeConfig { + ui32 NodeId; + TString InterconnectAddress; + TString GrpcHostname; + ui16 Port; + ui16 GrpcPort = 8080; + ui16 MbusPort = 0; + SOCKET Socket = -1; + SOCKET GrpcSocket = -1; + NYql::NProto::TDqConfig::TICSettings ICSettings = NYql::NProto::TDqConfig::TICSettings(); + TNameserverFactory NameserverFactory = [](const TIntrusivePtr<NActors::TTableNameserverSetup>& setup) { + return CreateNameserverTable(setup); }; -}; - -std::tuple<TString, TString> GetLocalAddress(const TString* hostname = nullptr); -std::tuple<TString, TString> GetUserToken(const TMaybe<TString>& user, const TMaybe<TString>& tokenFile); - -std::tuple<THolder<NActors::TActorSystemSetup>, TIntrusivePtr<NActors::NLog::TSettings>> BuildActorSetup( - ui32 nodeId, - TString interconnectAddress, - ui16 port, - SOCKET socket, - TVector<ui32> threads, - NMonitoring::TDynamicCounterPtr counters, - const TNameserverFactory& nameserverFactory, - const NYql::NProto::TDqConfig::TICSettings& icSettings = NYql::NProto::TDqConfig::TICSettings()); - -} // namespace NYql::NDqs +}; + +std::tuple<TString, TString> GetLocalAddress(const TString* hostname = nullptr); +std::tuple<TString, TString> GetUserToken(const TMaybe<TString>& user, const TMaybe<TString>& tokenFile); + +std::tuple<THolder<NActors::TActorSystemSetup>, TIntrusivePtr<NActors::NLog::TSettings>> BuildActorSetup( + ui32 nodeId, + TString interconnectAddress, + ui16 port, + SOCKET socket, + TVector<ui32> threads, + NMonitoring::TDynamicCounterPtr counters, + const TNameserverFactory& nameserverFactory, + const NYql::NProto::TDqConfig::TICSettings& icSettings = NYql::NProto::TDqConfig::TICSettings()); + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/service/service_node.cpp b/ydb/library/yql/providers/dq/service/service_node.cpp index 0374c8be35..cda24ec6cf 100644 --- a/ydb/library/yql/providers/dq/service/service_node.cpp +++ b/ydb/library/yql/providers/dq/service/service_node.cpp @@ -1,166 +1,166 @@ -#include "service_node.h" - +#include "service_node.h" + #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/providers/common/metrics/metrics_registry.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <ydb/library/yql/providers/dq/actors/execution_helpers.h> - + #include <library/cpp/grpc/server/actors/logger.h> #include <utility> - -namespace NYql { - using namespace NActors; + +namespace NYql { + using namespace NActors; using namespace NGrpc; - using namespace NYql::NDqs; - - class TGrpcExternalListener: public IExternalListener { - public: - TGrpcExternalListener(SOCKET socket) - : Socket(socket) - , Listener(MakeIntrusive<NInterconnect::TStreamSocket>(Socket)) - { - SetNonBlock(socket, true); - } - - void Init(std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> acceptor) override { - Acceptor = std::move(acceptor); - } - - private: - void Start() override { - YQL_LOG(DEBUG) << "Start GRPC Listener"; - Poller.Start(); - StartRead(); - } - - void Stop() override { - Poller.Stop(); - } - - void StartRead() { - YQL_LOG(TRACE) << "Read next GRPC event"; - Poller.StartRead(Listener, [&](const TIntrusivePtr<TSharedDescriptor>& ss) { - return Accept(ss); - }); - } - - TDelegate Accept(const TIntrusivePtr<TSharedDescriptor>& ss) - { - NInterconnect::TStreamSocket* socket = (NInterconnect::TStreamSocket*)ss.Get(); - int r = 0; - while (r >= 0) { - NInterconnect::TAddress address; - r = socket->Accept(address); - if (r >= 0) { - YQL_LOG(TRACE) << "New GRPC connection"; - grpc::experimental::ExternalConnectionAcceptor::NewConnectionParameters params; - SetNonBlock(r, true); - params.listener_fd = -1; // static_cast<int>(Socket); - params.fd = r; - Acceptor->HandleNewConnection(¶ms); - } else if (-r != EAGAIN && -r != EWOULDBLOCK) { - YQL_LOG(DEBUG) << "Unknown error code " + ToString(r); - } - } - - return [this] { - StartRead(); - }; - } - - SOCKET Socket; - TIntrusivePtr<NInterconnect::TStreamSocket> Listener; - NInterconnect::TPollerThreads Poller; - std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> Acceptor; - }; - - TServiceNode::TServiceNode( - const TServiceNodeConfig& config, - ui32 threads, - IMetricsRegistryPtr metricsRegistry) - : Config(config) - , Threads(threads) + using namespace NYql::NDqs; + + class TGrpcExternalListener: public IExternalListener { + public: + TGrpcExternalListener(SOCKET socket) + : Socket(socket) + , Listener(MakeIntrusive<NInterconnect::TStreamSocket>(Socket)) + { + SetNonBlock(socket, true); + } + + void Init(std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> acceptor) override { + Acceptor = std::move(acceptor); + } + + private: + void Start() override { + YQL_LOG(DEBUG) << "Start GRPC Listener"; + Poller.Start(); + StartRead(); + } + + void Stop() override { + Poller.Stop(); + } + + void StartRead() { + YQL_LOG(TRACE) << "Read next GRPC event"; + Poller.StartRead(Listener, [&](const TIntrusivePtr<TSharedDescriptor>& ss) { + return Accept(ss); + }); + } + + TDelegate Accept(const TIntrusivePtr<TSharedDescriptor>& ss) + { + NInterconnect::TStreamSocket* socket = (NInterconnect::TStreamSocket*)ss.Get(); + int r = 0; + while (r >= 0) { + NInterconnect::TAddress address; + r = socket->Accept(address); + if (r >= 0) { + YQL_LOG(TRACE) << "New GRPC connection"; + grpc::experimental::ExternalConnectionAcceptor::NewConnectionParameters params; + SetNonBlock(r, true); + params.listener_fd = -1; // static_cast<int>(Socket); + params.fd = r; + Acceptor->HandleNewConnection(¶ms); + } else if (-r != EAGAIN && -r != EWOULDBLOCK) { + YQL_LOG(DEBUG) << "Unknown error code " + ToString(r); + } + } + + return [this] { + StartRead(); + }; + } + + SOCKET Socket; + TIntrusivePtr<NInterconnect::TStreamSocket> Listener; + NInterconnect::TPollerThreads Poller; + std::unique_ptr<grpc::experimental::ExternalConnectionAcceptor> Acceptor; + }; + + TServiceNode::TServiceNode( + const TServiceNodeConfig& config, + ui32 threads, + IMetricsRegistryPtr metricsRegistry) + : Config(config) + , Threads(threads) , MetricsRegistry(std::move(metricsRegistry)) - { - std::tie(Setup, LogSettings) = BuildActorSetup( - Config.NodeId, - Config.InterconnectAddress, - Config.Port, - Config.Socket, - {Threads, 8}, - MetricsRegistry->GetSensors(), - Config.NameserverFactory, - Config.ICSettings); - } - + { + std::tie(Setup, LogSettings) = BuildActorSetup( + Config.NodeId, + Config.InterconnectAddress, + Config.Port, + Config.Socket, + {Threads, 8}, + MetricsRegistry->GetSensors(), + Config.NameserverFactory, + Config.ICSettings); + } + void TServiceNode::AddLocalService(TActorId actorId, const TActorSetupCmd& service) { - YQL_ENSURE(!ActorSystem); - Setup->LocalServices.emplace_back(actorId, service); - } - - NActors::TActorSystem* TServiceNode::StartActorSystem(void* appData) { - Y_VERIFY(!ActorSystem); - - ActorSystem = MakeHolder<NActors::TActorSystem>(Setup, appData, LogSettings); - ActorSystem->Start(); - - return ActorSystem.Get(); - } - + YQL_ENSURE(!ActorSystem); + Setup->LocalServices.emplace_back(actorId, service); + } + + NActors::TActorSystem* TServiceNode::StartActorSystem(void* appData) { + Y_VERIFY(!ActorSystem); + + ActorSystem = MakeHolder<NActors::TActorSystem>(Setup, appData, LogSettings); + ActorSystem->Start(); + + return ActorSystem.Get(); + } + void TServiceNode::StartService(const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories) { - class TCustomOption : public grpc::ServerBuilderOption { - public: - TCustomOption() { } - - void UpdateArguments(grpc::ChannelArguments *args) override { - args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 1); - } - - void UpdatePlugins(std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/) override - { } - }; - - YQL_LOG(INFO) << "Starting GRPC on " << Config.GrpcPort; - - IExternalListener::TPtr listener = nullptr; - if (Config.GrpcSocket >= 0) { - listener = MakeIntrusive<TGrpcExternalListener>(Config.GrpcSocket); - } - - auto options = TServerOptions() - // .SetHost(CurrentNode->Address) - .SetHost("[::]") - .SetPort(Config.GrpcPort) - .SetExternalListener(listener) - .SetWorkerThreads(2) - .SetGRpcMemoryQuotaBytes(1024 * 1024 * 1024) - .SetMaxMessageSize(1024 * 1024 * 256) - .SetMaxGlobalRequestInFlight(50000) - .SetUseAuth(false) - .SetKeepAliveEnable(true) + class TCustomOption : public grpc::ServerBuilderOption { + public: + TCustomOption() { } + + void UpdateArguments(grpc::ChannelArguments *args) override { + args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 1); + } + + void UpdatePlugins(std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/) override + { } + }; + + YQL_LOG(INFO) << "Starting GRPC on " << Config.GrpcPort; + + IExternalListener::TPtr listener = nullptr; + if (Config.GrpcSocket >= 0) { + listener = MakeIntrusive<TGrpcExternalListener>(Config.GrpcSocket); + } + + auto options = TServerOptions() + // .SetHost(CurrentNode->Address) + .SetHost("[::]") + .SetPort(Config.GrpcPort) + .SetExternalListener(listener) + .SetWorkerThreads(2) + .SetGRpcMemoryQuotaBytes(1024 * 1024 * 1024) + .SetMaxMessageSize(1024 * 1024 * 256) + .SetMaxGlobalRequestInFlight(50000) + .SetUseAuth(false) + .SetKeepAliveEnable(true) .SetKeepAliveIdleTimeoutTriggerSec(360) - .SetKeepAliveMaxProbeCount(3) - .SetKeepAliveProbeIntervalSec(1) - .SetServerBuilderMutator([](grpc::ServerBuilder& builder) { - builder.SetOption(std::make_unique<TCustomOption>()); + .SetKeepAliveMaxProbeCount(3) + .SetKeepAliveProbeIntervalSec(1) + .SetServerBuilderMutator([](grpc::ServerBuilder& builder) { + builder.SetOption(std::make_unique<TCustomOption>()); }) .SetLogger(CreateActorSystemLogger(*ActorSystem, 413)); // 413 - NKikimrServices::GRPC_SERVER - - Server = MakeHolder<TGRpcServer>(options); + + Server = MakeHolder<TGRpcServer>(options); Service = TIntrusivePtr<IGRpcService>(new TDqsGrpcService(*ActorSystem, MetricsRegistry->GetSensors(), dqTaskPreprocessorFactories)); - Server->AddService(Service); - Server->Start(); - } - - void TServiceNode::Stop(TDuration timeout) { - (static_cast<TDqsGrpcService*>(Service.Get()))->Stop().Wait(timeout); - - Server->Stop(); + Server->AddService(Service); + Server->Start(); + } + + void TServiceNode::Stop(TDuration timeout) { + (static_cast<TDqsGrpcService*>(Service.Get()))->Stop().Wait(timeout); + + Server->Stop(); for (auto id : ActorIds) { ActorSystem->Send(id, new NActors::TEvents::TEvPoison); } - ActorSystem->Stop(); - } -} // namespace NYql + ActorSystem->Stop(); + } +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/service/service_node.h b/ydb/library/yql/providers/dq/service/service_node.h index 4a04e068f2..32f904901d 100644 --- a/ydb/library/yql/providers/dq/service/service_node.h +++ b/ydb/library/yql/providers/dq/service/service_node.h @@ -1,13 +1,13 @@ -#pragma once - -#include "grpc_service.h" -#include "interconnect_helpers.h" - +#pragma once + +#include "grpc_service.h" +#include "interconnect_helpers.h" + #include <ydb/library/yql/providers/common/metrics/metrics_registry.h> #include <ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h> - + #include <ydb/library/yql/minikql/mkql_function_registry.h> - + #include <library/cpp/actors/core/executor_pool_basic.h> #include <library/cpp/actors/core/scheduler_basic.h> #include <library/cpp/actors/interconnect/interconnect.h> @@ -15,34 +15,34 @@ #include <library/cpp/actors/interconnect/interconnect_tcp_proxy.h> #include <library/cpp/actors/interconnect/interconnect_tcp_server.h> #include <library/cpp/actors/interconnect/poller_actor.h> - -namespace NYql { - class TServiceNode { - public: - TServiceNode( - const NDqs::TServiceNodeConfig& config, - ui32 threads, - IMetricsRegistryPtr metricsRegistry); - + +namespace NYql { + class TServiceNode { + public: + TServiceNode( + const NDqs::TServiceNodeConfig& config, + ui32 threads, + IMetricsRegistryPtr metricsRegistry); + void AddLocalService(NActors::TActorId actorId, const NActors::TActorSetupCmd& service); - NActors::TActorSystem* StartActorSystem(void* appData = nullptr); + NActors::TActorSystem* StartActorSystem(void* appData = nullptr); void StartService(const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories); - - void Stop(TDuration time = TDuration::Max()); - - NActors::TActorSystemSetup* GetSetup() const { - return Setup.Get(); - } - - private: - NDqs::TServiceNodeConfig Config; - ui32 Threads; + + void Stop(TDuration time = TDuration::Max()); + + NActors::TActorSystemSetup* GetSetup() const { + return Setup.Get(); + } + + private: + NDqs::TServiceNodeConfig Config; + ui32 Threads; IMetricsRegistryPtr MetricsRegistry; - THolder<NActors::TActorSystemSetup> Setup; - TIntrusivePtr<NActors::NLog::TSettings> LogSettings; - THolder<NActors::TActorSystem> ActorSystem; + THolder<NActors::TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<NActors::TActorSystem> ActorSystem; TVector<NActors::TActorId> ActorIds; THolder<NGrpc::TGRpcServer> Server; TIntrusivePtr<NGrpc::IGRpcService> Service; - }; -} + }; +} diff --git a/ydb/library/yql/providers/dq/service/ya.make b/ydb/library/yql/providers/dq/service/ya.make index 67c6acefae..7af208b9cd 100644 --- a/ydb/library/yql/providers/dq/service/ya.make +++ b/ydb/library/yql/providers/dq/service/ya.make @@ -4,8 +4,8 @@ OWNER(g:yql) SRCS( grpc_service.cpp - grpc_session.cpp - service_node.cpp + grpc_session.cpp + service_node.cpp interconnect_helpers.cpp ) diff --git a/ydb/library/yql/providers/dq/task_runner/file_cache.h b/ydb/library/yql/providers/dq/task_runner/file_cache.h index bce078f522..33e707334f 100644 --- a/ydb/library/yql/providers/dq/task_runner/file_cache.h +++ b/ydb/library/yql/providers/dq/task_runner/file_cache.h @@ -1,36 +1,36 @@ -#pragma once - -#include <util/generic/string.h> -#include <util/generic/hash.h> -#include <util/generic/guid.h> -#include <util/generic/maybe.h> -#include <util/system/mutex.h> - +#pragma once + +#include <util/generic/string.h> +#include <util/generic/hash.h> +#include <util/generic/guid.h> +#include <util/generic/maybe.h> +#include <util/system/mutex.h> + #include <atomic> -namespace NYql { - -class IFileCache: public TSimpleRefCount<IFileCache> { -public: - using TPtr = TIntrusivePtr<IFileCache>; - - virtual ~IFileCache() = default; - - virtual void Clear() { } - - virtual void AddFile(const TString& path, const TString& objectId) = 0; - - virtual TMaybe<TString> FindFile(const TString& objectId) = 0; - - virtual bool Contains(const TString& objectId) = 0; - - virtual void Walk(const std::function<void(const TString& objectId)>&) = 0; - - virtual ui64 FreeDiskSize() = 0; - - virtual ui64 UsedDiskSize() = 0; - - virtual TString GetDir() = 0; -}; - -} // namespace NYql +namespace NYql { + +class IFileCache: public TSimpleRefCount<IFileCache> { +public: + using TPtr = TIntrusivePtr<IFileCache>; + + virtual ~IFileCache() = default; + + virtual void Clear() { } + + virtual void AddFile(const TString& path, const TString& objectId) = 0; + + virtual TMaybe<TString> FindFile(const TString& objectId) = 0; + + virtual bool Contains(const TString& objectId) = 0; + + virtual void Walk(const std::function<void(const TString& objectId)>&) = 0; + + virtual ui64 FreeDiskSize() = 0; + + virtual ui64 UsedDiskSize() = 0; + + virtual TString GetDir() = 0; +}; + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h b/ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h index 63e28928f9..e0ac500180 100644 --- a/ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h +++ b/ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h @@ -1,30 +1,30 @@ -#pragma once - -namespace NYql::NDqs { - -struct ITaskRunnerInvoker: public TThrRefBase { - using TPtr = TIntrusivePtr<ITaskRunnerInvoker>; - virtual void Invoke(const std::function<void(void)>& f) = 0; - virtual bool IsLocal() { return false; } -}; - -struct TTaskRunnerInvoker: public ITaskRunnerInvoker { - void Invoke(const std::function<void(void)>& f) override { - f(); - } - - bool IsLocal() override { return true; } -}; - -struct ITaskRunnerInvokerFactory: public TThrRefBase { - using TPtr = TIntrusivePtr<ITaskRunnerInvokerFactory>; - virtual ITaskRunnerInvoker::TPtr Create() = 0; -}; - -class TTaskRunnerInvokerFactory: public ITaskRunnerInvokerFactory { - ITaskRunnerInvoker::TPtr Create() override { - return new TTaskRunnerInvoker(); - } -}; - -} // namespace NYql::NDq +#pragma once + +namespace NYql::NDqs { + +struct ITaskRunnerInvoker: public TThrRefBase { + using TPtr = TIntrusivePtr<ITaskRunnerInvoker>; + virtual void Invoke(const std::function<void(void)>& f) = 0; + virtual bool IsLocal() { return false; } +}; + +struct TTaskRunnerInvoker: public ITaskRunnerInvoker { + void Invoke(const std::function<void(void)>& f) override { + f(); + } + + bool IsLocal() override { return true; } +}; + +struct ITaskRunnerInvokerFactory: public TThrRefBase { + using TPtr = TIntrusivePtr<ITaskRunnerInvokerFactory>; + virtual ITaskRunnerInvoker::TPtr Create() = 0; +}; + +class TTaskRunnerInvokerFactory: public ITaskRunnerInvokerFactory { + ITaskRunnerInvoker::TPtr Create() override { + return new TTaskRunnerInvoker(); + } +}; + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp index ab70a32ce6..ab8769f44e 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp @@ -1,6 +1,6 @@ #include "tasks_runner_local.h" #include "file_cache.h" - + #include <ydb/library/yql/providers/dq/counters/counters.h> #include <ydb/library/yql/dq/runtime/dq_input_channel.h> #include <ydb/library/yql/dq/runtime/dq_output_channel.h> @@ -12,139 +12,139 @@ #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/backtrace/backtrace.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <library/cpp/yson/node/node.h> #include <library/cpp/yson/node/node_io.h> #include <library/cpp/svnversion/svnversion.h> -#include <util/system/env.h> -#include <util/stream/file.h> +#include <util/system/env.h> +#include <util/stream/file.h> #include <util/generic/size_literals.h> - - + + namespace NYql::NTaskRunnerProxy { - -using namespace NKikimr; -using namespace NDq; - -#define ADD_COUNTER(name) \ - if (stats->name) { \ - QueryStat.AddCounter(QueryStat.GetCounterName("TaskRunner", labels, #name), stats->name); \ - } - -#define ADD_TIME_COUNTER(name) \ - if (stats->name) { \ - QueryStat.AddTimeCounter(QueryStat.GetCounterName("TaskRunner", labels, #name), stats->name); \ - } - -class TLocalInputChannel: public IInputChannel { -public: - TLocalInputChannel(const IDqInputChannel::TPtr& channel, ui64 taskId, ui64 channelId, TCounters* queryStat) - : TaskId(taskId) - , ChannelId(channelId) - , Channel(channel) - , QueryStat(*queryStat) - , Stats(channelId) - { } - - void Push(NDqProto::TData&& data) override { - Channel->Push(std::move(data)); - } - - i64 GetFreeSpace() override { - return Channel->GetFreeSpace(); - } - - void Finish() override { - Channel->Finish(); - UpdateInputChannelStats(); - } - -private: - void UpdateInputChannelStats() - { - QueryStat.AddInputChannelStats(*Channel->GetStats(), Stats, TaskId, ChannelId); - } - - ui64 TaskId; - ui64 ChannelId; - IDqInputChannel::TPtr Channel; - TCounters& QueryStat; - TDqInputChannelStats Stats; -}; - -class TLocalOutputChannel : public IOutputChannel { -public: - TLocalOutputChannel(const IDqOutputChannel::TPtr channel, ui64 taskId, ui64 channelId, TCounters* queryStat) - : TaskId(taskId) - , ChannelId(channelId) - , Channel(channel) - , QueryStat(*queryStat) - , Stats(channelId) - { } - - [[nodiscard]] + +using namespace NKikimr; +using namespace NDq; + +#define ADD_COUNTER(name) \ + if (stats->name) { \ + QueryStat.AddCounter(QueryStat.GetCounterName("TaskRunner", labels, #name), stats->name); \ + } + +#define ADD_TIME_COUNTER(name) \ + if (stats->name) { \ + QueryStat.AddTimeCounter(QueryStat.GetCounterName("TaskRunner", labels, #name), stats->name); \ + } + +class TLocalInputChannel: public IInputChannel { +public: + TLocalInputChannel(const IDqInputChannel::TPtr& channel, ui64 taskId, ui64 channelId, TCounters* queryStat) + : TaskId(taskId) + , ChannelId(channelId) + , Channel(channel) + , QueryStat(*queryStat) + , Stats(channelId) + { } + + void Push(NDqProto::TData&& data) override { + Channel->Push(std::move(data)); + } + + i64 GetFreeSpace() override { + return Channel->GetFreeSpace(); + } + + void Finish() override { + Channel->Finish(); + UpdateInputChannelStats(); + } + +private: + void UpdateInputChannelStats() + { + QueryStat.AddInputChannelStats(*Channel->GetStats(), Stats, TaskId, ChannelId); + } + + ui64 TaskId; + ui64 ChannelId; + IDqInputChannel::TPtr Channel; + TCounters& QueryStat; + TDqInputChannelStats Stats; +}; + +class TLocalOutputChannel : public IOutputChannel { +public: + TLocalOutputChannel(const IDqOutputChannel::TPtr channel, ui64 taskId, ui64 channelId, TCounters* queryStat) + : TaskId(taskId) + , ChannelId(channelId) + , Channel(channel) + , QueryStat(*queryStat) + , Stats(channelId) + { } + + [[nodiscard]] NDqProto::TPopResponse Pop(NDqProto::TData& data, ui64 bytes) override { - NDqProto::TPopResponse response; - response.SetResult(Channel->Pop(data, bytes)); - if (Channel->IsFinished()) { - UpdateOutputChannelStats(); - QueryStat.FlushCounters(response); - } - return response; - } - + NDqProto::TPopResponse response; + response.SetResult(Channel->Pop(data, bytes)); + if (Channel->IsFinished()) { + UpdateOutputChannelStats(); + QueryStat.FlushCounters(response); + } + return response; + } + bool IsFinished() const override { - return Channel->IsFinished(); - } - -private: - void UpdateOutputChannelStats() - { - QueryStat.AddOutputChannelStats(*Channel->GetStats(), Stats, TaskId, ChannelId); - } - - ui64 TaskId; - ui64 ChannelId; + return Channel->IsFinished(); + } + +private: + void UpdateOutputChannelStats() + { + QueryStat.AddOutputChannelStats(*Channel->GetStats(), Stats, TaskId, ChannelId); + } + + ui64 TaskId; + ui64 ChannelId; IDqOutputChannel::TPtr Channel; - TCounters& QueryStat; - TDqOutputChannelStats Stats; -}; - + TCounters& QueryStat; + TDqOutputChannelStats Stats; +}; + class TLocalTaskRunner: public ITaskRunner { -public: - TLocalTaskRunner(const NDqProto::TDqTask& task, TIntrusivePtr<IDqTaskRunner> runner) +public: + TLocalTaskRunner(const NDqProto::TDqTask& task, TIntrusivePtr<IDqTaskRunner> runner) : Task(task) , Runner(runner) - { } - - ~TLocalTaskRunner() - { } - - i32 GetProtocolVersion() override { - return 1; - } - + { } + + ~TLocalTaskRunner() + { } + + i32 GetProtocolVersion() override { + return 1; + } + ui64 GetTaskId() const override { return Task.GetId(); - } - - NYql::NDqProto::TPrepareResponse Prepare() override { - NYql::NDqProto::TPrepareResponse ret; + } + + NYql::NDqProto::TPrepareResponse Prepare() override { + NYql::NDqProto::TPrepareResponse ret; Runner->Prepare(Task, DefaultMemoryLimits()); - return ret; - } - - NYql::NDqProto::TRunResponse Run() override { - auto status = Runner->Run(); - NYql::NDqProto::TRunResponse response; - response.SetResult(static_cast<ui32>(status)); - - if (status == NDq::ERunStatus::Finished) { - UpdateStats(); - QueryStat.FlushCounters(response); - } - return response; + return ret; + } + + NYql::NDqProto::TRunResponse Run() override { + auto status = Runner->Run(); + NYql::NDqProto::TRunResponse response; + response.SetResult(static_cast<ui32>(status)); + + if (status == NDq::ERunStatus::Finished) { + UpdateStats(); + QueryStat.FlushCounters(response); + } + return response; } IInputChannel::TPtr GetInputChannel(ui64 channelId) override { @@ -155,26 +155,26 @@ public: return new TLocalOutputChannel(Runner->GetOutputChannel(channelId), Task.GetId(), channelId, &QueryStat); } - IDqSource::TPtr GetSource(ui64 index) override { - return Runner->GetSource(index); - } - - IDqSink::TPtr GetSink(ui64 index) override { - return Runner->GetSink(index); - } - - const THashMap<TString,TString>& GetTaskParams() const override { - return Runner->GetTaskParams(); - } - - const THashMap<TString,TString>& GetSecureParams() const override { - return Runner->GetSecureParams(); - } - + IDqSource::TPtr GetSource(ui64 index) override { + return Runner->GetSource(index); + } + + IDqSink::TPtr GetSink(ui64 index) override { + return Runner->GetSink(index); + } + + const THashMap<TString,TString>& GetTaskParams() const override { + return Runner->GetTaskParams(); + } + + const THashMap<TString,TString>& GetSecureParams() const override { + return Runner->GetSecureParams(); + } + const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override { - return Runner->GetTypeEnv(); - } - + return Runner->GetTypeEnv(); + } + const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override { return Runner->GetHolderFactory(); } @@ -187,82 +187,82 @@ public: return Runner->IsAllocatorAttached(); } - TStatus GetStatus() override { - return {0, ""}; - } - -private: - void UpdateStats() { - QueryStat.AddTaskRunnerStats(*Runner->GetStats(), Stats, Task.GetId()); - } - + TStatus GetStatus() override { + return {0, ""}; + } + +private: + void UpdateStats() { + QueryStat.AddTaskRunnerStats(*Runner->GetStats(), Stats, Task.GetId()); + } + NDqProto::TDqTask Task; TIntrusivePtr<IDqTaskRunner> Runner; - TCounters QueryStat; - TDqTaskRunnerStats Stats; -}; - -/*______________________________________________________________________________________________*/ - -class TAbstractFactory: public IProxyFactory { -public: + TCounters QueryStat; + TDqTaskRunnerStats Stats; +}; + +/*______________________________________________________________________________________________*/ + +class TAbstractFactory: public IProxyFactory { +public: TAbstractFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory) - : DeterministicMode(!!GetEnv("YQL_DETERMINISTIC_MODE")) - , RandomProvider( - DeterministicMode - ? CreateDeterministicRandomProvider(1) - : CreateDefaultRandomProvider()) - , TimeProvider( - DeterministicMode - ? CreateDeterministicTimeProvider(10000000) - : CreateDefaultTimeProvider()) - , FunctionRegistry(functionRegistry) + : DeterministicMode(!!GetEnv("YQL_DETERMINISTIC_MODE")) + , RandomProvider( + DeterministicMode + ? CreateDeterministicRandomProvider(1) + : CreateDefaultRandomProvider()) + , TimeProvider( + DeterministicMode + ? CreateDeterministicTimeProvider(10000000) + : CreateDefaultTimeProvider()) + , FunctionRegistry(functionRegistry) , TaskTransformFactory(std::move(taskTransformFactory)) - { - ExecutionContext.FuncRegistry = FunctionRegistry; + { + ExecutionContext.FuncRegistry = FunctionRegistry; ExecutionContext.ComputationFactory = compFactory; - ExecutionContext.RandomProvider = RandomProvider.Get(); - ExecutionContext.TimeProvider = TimeProvider.Get(); - } - -protected: - bool DeterministicMode; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; + ExecutionContext.RandomProvider = RandomProvider.Get(); + ExecutionContext.TimeProvider = TimeProvider.Get(); + } + +protected: + bool DeterministicMode; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry; TTaskTransformFactory TaskTransformFactory; - + NDq::TDqTaskRunnerContext ExecutionContext; -}; - +}; + /*______________________________________________________________________________________________*/ -class TLocalFactory: public TAbstractFactory { -public: +class TLocalFactory: public TAbstractFactory { +public: TLocalFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, bool terminateOnError) : TAbstractFactory(functionRegistry, compFactory, taskTransformFactory) , TerminateOnError(terminateOnError) - { } - - ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& task, const TString& traceId) override { - return new TLocalTaskRunner(task, Get(task, traceId)); + { } + + ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& task, const TString& traceId) override { + return new TLocalTaskRunner(task, Get(task, traceId)); } - TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& task, const TString& traceId) override { - Y_UNUSED(traceId); + TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& task, const TString& traceId) override { + Y_UNUSED(traceId); NDq::TDqTaskRunnerSettings settings; settings.TerminateOnError = TerminateOnError; - settings.CollectBasicStats = true; - settings.CollectProfileStats = true; + settings.CollectBasicStats = true; + settings.CollectProfileStats = true; settings.AllowGeneratorsInUnboxedValues = true; - - Yql::DqsProto::TTaskMeta taskMeta; - task.GetMeta().UnpackTo(&taskMeta); - - for (const auto& s : taskMeta.GetSettings()) { + + Yql::DqsProto::TTaskMeta taskMeta; + task.GetMeta().UnpackTo(&taskMeta); + + for (const auto& s : taskMeta.GetSettings()) { if ("OptLLVM" == s.GetName()) settings.OptLLVM = s.GetValue(); } @@ -276,17 +276,17 @@ public: auto ctx = ExecutionContext; ctx.FuncProvider = TaskTransformFactory(settings.TaskParams, ctx.FuncRegistry); return MakeDqTaskRunner(ctx, settings, { }); - } + } private: const bool TerminateOnError; -}; - +}; + IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, bool terminateOnError) -{ +{ return new TLocalFactory(functionRegistry, compFactory, taskTransformFactory, terminateOnError); -} - +} + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h index b0cd00f04b..1702041a74 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include "tasks_runner_proxy.h" - + #include <ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h> - + namespace NYql::NTaskRunnerProxy { - + IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, bool terminateOnError ); - + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp index c1abf499af..cdcbd32ef5 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp @@ -1,5 +1,5 @@ #include "tasks_runner_pipe.h" - + #include <ydb/library/yql/dq/runtime/dq_input_channel.h> #include <ydb/library/yql/dq/runtime/dq_output_channel.h> #include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> @@ -10,69 +10,69 @@ #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/backtrace/backtrace.h> #include <ydb/library/yql/utils/yql_panic.h> - + #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h> - + #include <library/cpp/yson/node/node.h> #include <library/cpp/yson/node/node_io.h> #include <library/cpp/svnversion/svnversion.h> -#include <library/cpp/threading/task_scheduler/task_scheduler.h> - -#include <util/system/shellcommand.h> -#include <util/system/thread.h> -#include <util/system/fs.h> -#include <util/stream/file.h> -#include <util/stream/pipe.h> +#include <library/cpp/threading/task_scheduler/task_scheduler.h> + +#include <util/system/shellcommand.h> +#include <util/system/thread.h> +#include <util/system/fs.h> +#include <util/stream/file.h> +#include <util/stream/pipe.h> #include <util/generic/size_literals.h> #include <util/string/cast.h> - + namespace NYql::NTaskRunnerProxy { - -const TString WorkingDirectoryParamName = "TaskRunnerProxy.WorkingDirectory"; -const TString WorkingDirectoryDontInitParamName = "TaskRunnerProxy.WorkingDirectoryDontInit"; -const TString UseMetaParamName = "TaskRunnerProxy.UseMeta"; // COMPAT(aozeritsky) - + +const TString WorkingDirectoryParamName = "TaskRunnerProxy.WorkingDirectory"; +const TString WorkingDirectoryDontInitParamName = "TaskRunnerProxy.WorkingDirectoryDontInit"; +const TString UseMetaParamName = "TaskRunnerProxy.UseMeta"; // COMPAT(aozeritsky) + using namespace NKikimr; using namespace NDq; - -#ifndef _win_ -extern "C" int fork(void); -extern "C" int dup2(int oldfd, int newfd); -extern "C" int close(int); -extern "C" int pipe(int pipefd[2]); -extern "C" int kill(int pid, int sig); -extern "C" int waitpid(int pid, int* status, int options); -#endif - -template<typename T> -void FromProto(TDqOutputChannelStats* s, const T& f) -{ - s->ChannelId = f.GetChannelId(); - s->Chunks = f.GetChunks(); - s->Bytes = f.GetBytes(); - s->RowsIn = f.GetRowsIn(); - s->RowsOut = f.GetRowsOut(); - s->MaxMemoryUsage = f.GetMaxMemoryUsage(); + +#ifndef _win_ +extern "C" int fork(void); +extern "C" int dup2(int oldfd, int newfd); +extern "C" int close(int); +extern "C" int pipe(int pipefd[2]); +extern "C" int kill(int pid, int sig); +extern "C" int waitpid(int pid, int* status, int options); +#endif + +template<typename T> +void FromProto(TDqOutputChannelStats* s, const T& f) +{ + s->ChannelId = f.GetChannelId(); + s->Chunks = f.GetChunks(); + s->Bytes = f.GetBytes(); + s->RowsIn = f.GetRowsIn(); + s->RowsOut = f.GetRowsOut(); + s->MaxMemoryUsage = f.GetMaxMemoryUsage(); //s->StartTs = TInstant::MilliSeconds(f.GetStartTs()); //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs()); -} - -template<typename T> -void FromProto(TDqInputChannelStats* s, const T& f) -{ - s->ChannelId = f.GetChannelId(); - s->Chunks = f.GetChunks(); - s->Bytes = f.GetBytes(); - s->RowsIn = f.GetRowsIn(); - s->RowsOut = f.GetRowsOut(); - s->MaxMemoryUsage = f.GetMaxMemoryUsage(); +} + +template<typename T> +void FromProto(TDqInputChannelStats* s, const T& f) +{ + s->ChannelId = f.GetChannelId(); + s->Chunks = f.GetChunks(); + s->Bytes = f.GetBytes(); + s->RowsIn = f.GetRowsIn(); + s->RowsOut = f.GetRowsOut(); + s->MaxMemoryUsage = f.GetMaxMemoryUsage(); //s->StartTs = TInstant::MilliSeconds(f.GetStartTs()); //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs()); s->DeserializationTime = TDuration::MicroSeconds(f.GetDeserializationTimeUs()); -} - -template<typename T> +} + +template<typename T> void FromProto(TDqSourceStats* s, const T& f) { s->InputIndex = f.GetInputIndex(); @@ -86,18 +86,18 @@ void FromProto(TDqSourceStats* s, const T& f) } template<typename T> -void FromProto(TDqSinkStats* s, const T& f) -{ - s->Chunks = f.GetChunks(); - s->Bytes = f.GetBytes(); - s->RowsIn = f.GetRowsIn(); - s->RowsOut = f.GetRowsOut(); - s->MaxMemoryUsage = f.GetMaxMemoryUsage(); -} - -template<typename T> -void FromProto(TDqTaskRunnerStats* s, const T& f) -{ +void FromProto(TDqSinkStats* s, const T& f) +{ + s->Chunks = f.GetChunks(); + s->Bytes = f.GetBytes(); + s->RowsIn = f.GetRowsIn(); + s->RowsOut = f.GetRowsOut(); + s->MaxMemoryUsage = f.GetMaxMemoryUsage(); +} + +template<typename T> +void FromProto(TDqTaskRunnerStats* s, const T& f) +{ //s->StartTs = TInstant::MilliSeconds(f.GetStartTs()); //s->FinishTs = TInstant::MilliSeconds(f.GetFinishTs()); s->BuildCpuTime = TDuration::MicroSeconds(f.GetBuildCpuTimeUs()); @@ -108,7 +108,7 @@ void FromProto(TDqTaskRunnerStats* s, const T& f) //s->TotalTime = TDuration::MilliSeconds(f.GetTotalTime()); s->WaitTime = TDuration::MicroSeconds(f.GetWaitTimeUs()); s->WaitOutputTime = TDuration::MicroSeconds(f.GetWaitOutputTimeUs()); - + //s->MkqlTotalNodes = f.GetMkqlTotalNodes(); //s->MkqlCodegenFunctions = f.GetMkqlCodegenFunctions(); //s->CodeGenTotalInstructions = f.GetCodeGenTotalInstructions(); @@ -116,559 +116,559 @@ void FromProto(TDqTaskRunnerStats* s, const T& f) //s->CodeGenFullTime = f.GetCodeGenFullTime(); //s->CodeGenFinalizeTime = f.GetCodeGenFinalizeTime(); //s->CodeGenModulePassTime = f.GetCodeGenModulePassTime(); - - for (const auto& input : f.GetInputChannels()) { - FromProto(const_cast<TDqInputChannelStats*>(s->InputChannels[input.GetChannelId()]), input); - } - - for (const auto& output : f.GetOutputChannels()) { - FromProto(const_cast<TDqOutputChannelStats*>(s->OutputChannels[output.GetChannelId()]), output); - } -} - -class TChildProcess: private TNonCopyable { -public: - TChildProcess(const TString& exeName, const TVector<TString>& args, const THashMap<TString, TString>& env, const TString& workDir) - : ExeName(exeName) - , Args(args) - , Env(env) - , WorkDir(workDir) - { - NFs::MakeDirectory(WorkDir); - } - - virtual ~TChildProcess() { - try { - NFs::RemoveRecursive(WorkDir); - } catch (...) { } - } - - virtual TString ExternalWorkDir() { - return WorkDir; - } - - virtual TString InternalWorkDir() { - return WorkDir; - } - - void Run() - { -#ifdef _win_ - Y_VERIFY(false); -#else - int input[2]; - int output[2]; - int error[2]; - - Y_VERIFY(pipe(input) == 0); - Y_VERIFY(pipe(output) == 0); - Y_VERIFY(pipe(error) == 0); - - Pid = fork(); - Y_VERIFY(Pid >= 0); - - if (Pid == 0) { - try { - close(input[1]); - close(output[0]); - close(error[0]); - - if (dup2(input[0], 0) == -1) { - ythrow TSystemError() << "Cannot dup2 input"; - } // stdin - if (dup2(output[1], 1) == -1) { - ythrow TSystemError() << "Cannot dup2 output"; - } // stdout - if (dup2(error[1], 2) == -1) { - ythrow TSystemError() << "Cannot dup2 error"; - } // stderr - - Exec(); - } catch (const yexception& ex) { - Cerr << ex.what() << Endl; - } - - _exit(127); - } - - close(input[0]); - close(output[1]); - close(error[1]); - - Stdin = MakeHolder<TPipedOutput>(input[1]); - Stdout = MakeHolder<TPipedInput>(output[0]); - Stderr = MakeHolder<TPipedInput>(error[0]); -#endif - } - - virtual void Kill() { -#ifndef _win_ - kill(Pid, 9); -#endif - } - - bool IsAlive() { -#ifdef _win_ - return true; -#else - int status; - YQL_LOG(DEBUG) << "Check Pid " << Pid; - return waitpid(Pid, &status, WNOHANG) <= 0; -#endif - } - - int Wait(TDuration timeout = TDuration::Seconds(5)) { - int status; - int ret; -#ifndef _win_ - TInstant start = TInstant::Now(); - while ((ret = waitpid(Pid, &status, WNOHANG)) == 0 && TInstant::Now() - start < timeout) { - Sleep(TDuration::MilliSeconds(10)); - } - if (ret <= 0) { - kill(Pid, 9); - waitpid(Pid, &status, 0); - } -#endif - return status; - } - - IOutputStream& GetStdin() { - return *Stdin; - } - - IInputStream& GetStdout() { - return *Stdout; - } - - IInputStream& GetStderr() { - return *Stderr; - } - -protected: - TString ExeName; - const TVector<TString> Args; - const THashMap<TString, TString> Env; - const TString WorkDir; - - THolder<TPipedOutput> Stdin; - THolder<TPipedInput> Stdout; - THolder<TPipedInput> Stderr; - + + for (const auto& input : f.GetInputChannels()) { + FromProto(const_cast<TDqInputChannelStats*>(s->InputChannels[input.GetChannelId()]), input); + } + + for (const auto& output : f.GetOutputChannels()) { + FromProto(const_cast<TDqOutputChannelStats*>(s->OutputChannels[output.GetChannelId()]), output); + } +} + +class TChildProcess: private TNonCopyable { +public: + TChildProcess(const TString& exeName, const TVector<TString>& args, const THashMap<TString, TString>& env, const TString& workDir) + : ExeName(exeName) + , Args(args) + , Env(env) + , WorkDir(workDir) + { + NFs::MakeDirectory(WorkDir); + } + + virtual ~TChildProcess() { + try { + NFs::RemoveRecursive(WorkDir); + } catch (...) { } + } + + virtual TString ExternalWorkDir() { + return WorkDir; + } + + virtual TString InternalWorkDir() { + return WorkDir; + } + + void Run() + { +#ifdef _win_ + Y_VERIFY(false); +#else + int input[2]; + int output[2]; + int error[2]; + + Y_VERIFY(pipe(input) == 0); + Y_VERIFY(pipe(output) == 0); + Y_VERIFY(pipe(error) == 0); + + Pid = fork(); + Y_VERIFY(Pid >= 0); + + if (Pid == 0) { + try { + close(input[1]); + close(output[0]); + close(error[0]); + + if (dup2(input[0], 0) == -1) { + ythrow TSystemError() << "Cannot dup2 input"; + } // stdin + if (dup2(output[1], 1) == -1) { + ythrow TSystemError() << "Cannot dup2 output"; + } // stdout + if (dup2(error[1], 2) == -1) { + ythrow TSystemError() << "Cannot dup2 error"; + } // stderr + + Exec(); + } catch (const yexception& ex) { + Cerr << ex.what() << Endl; + } + + _exit(127); + } + + close(input[0]); + close(output[1]); + close(error[1]); + + Stdin = MakeHolder<TPipedOutput>(input[1]); + Stdout = MakeHolder<TPipedInput>(output[0]); + Stderr = MakeHolder<TPipedInput>(error[0]); +#endif + } + + virtual void Kill() { +#ifndef _win_ + kill(Pid, 9); +#endif + } + + bool IsAlive() { +#ifdef _win_ + return true; +#else + int status; + YQL_LOG(DEBUG) << "Check Pid " << Pid; + return waitpid(Pid, &status, WNOHANG) <= 0; +#endif + } + + int Wait(TDuration timeout = TDuration::Seconds(5)) { + int status; + int ret; +#ifndef _win_ + TInstant start = TInstant::Now(); + while ((ret = waitpid(Pid, &status, WNOHANG)) == 0 && TInstant::Now() - start < timeout) { + Sleep(TDuration::MilliSeconds(10)); + } + if (ret <= 0) { + kill(Pid, 9); + waitpid(Pid, &status, 0); + } +#endif + return status; + } + + IOutputStream& GetStdin() { + return *Stdin; + } + + IInputStream& GetStdout() { + return *Stdout; + } + + IInputStream& GetStderr() { + return *Stderr; + } + +protected: + TString ExeName; + const TVector<TString> Args; + const THashMap<TString, TString> Env; + const TString WorkDir; + + THolder<TPipedOutput> Stdin; + THolder<TPipedInput> Stdout; + THolder<TPipedInput> Stderr; + int Pid = -1; - - virtual void Exec() { - char ** args; - args = new char*[Args.size() + 1]; - ui32 i; - for (i = 0; i < Args.size(); ++i) { - args[i] = (char*) Args[i].c_str(); - } - args[i] = nullptr; - - for (i = 3; i < 32768; ++i) { - close(i); - } - - char** env; - env = new char*[Env.size() + 1]; - i = 0; - for (const auto& [k, v] : Env) { - env[i++] = (char*)(k + "=" + v).c_str(); - } - env[i++] = nullptr; - - if (!WorkDir.empty()) { - NFs::SetCurrentWorkingDirectory(WorkDir); - } - - if (execve(ExeName.c_str(), (char*const*)args, (char*const*)env) == -1) { - ythrow TSystemError() << "Cannot execl"; - } - } -}; - -/*______________________________________________________________________________________________*/ - -struct TProcessHolder { - TProcessHolder() - : Watcher(MakeHolder<TThread>([this] () { Watch(); })) - { - Running.test_and_set(); - Watcher->Start(); - } - - ~TProcessHolder() - { - Running.clear(); - Watcher->Join(); - } - - i64 Size() { - TGuard<TMutex> lock(Mutex); - return Processes.size(); - } - - void Put(const TString& key, THolder<TChildProcess>process) { - TGuard<TMutex> lock(Mutex); - Processes.emplace_back(key, std::move(process)); - } - - THolder<TChildProcess> Acquire(const TString& key, TList<THolder<TChildProcess>>* stopList) { - TGuard<TMutex> lock(Mutex); - THolder<TChildProcess> result; - while (!Processes.empty()) { - auto first = std::move(Processes.front()); - Processes.pop_front(); - if (first.first == key) { - result = std::move(first.second); - break; - } - stopList->push_back(std::move(first.second)); - } - return result; - } - - void Watch() { - while (Running.test()) { - TList<THolder<TChildProcess>> stopList; - { - TGuard<TMutex> lock(Mutex); - auto it = Processes.begin(); - while (it != Processes.end()) { - if (!it->second->IsAlive()) { - YQL_LOG(DEBUG) << "Remove dead process"; - stopList.emplace_back(std::move(it->second)); - it = Processes.erase(it); - } else { - ++it; - } - } - } - - for (const auto& job : stopList) { - job->Kill(); - job->Wait(TDuration::Seconds(1)); - } - Sleep(TDuration::MilliSeconds(1000)); - } - } - - THolder<TThread> Watcher; - std::atomic_flag Running; - - TMutex Mutex; - TList<std::pair<TString, THolder<TChildProcess>>> Processes; -}; - -struct TPortoSettings { - bool Enable; - ui64 MemoryLimit; - TString Layer; - TString ContainerNamePrefix; - - bool operator == (const TPortoSettings& o) const { - return Enable == o.Enable && MemoryLimit == o.MemoryLimit && Layer == o.Layer; - } - - bool operator != (const TPortoSettings& o) const { - return ! operator == (o); - } - - TString ToString() const { - return TStringBuilder() << Enable << "," << MemoryLimit << "," << Layer; - } -}; - -class TPortoProcess: public TChildProcess -{ -public: - TPortoProcess(const TString& portoCtl, const TString& exeName, const TVector<TString>& args, const THashMap<TString, TString>& env, const TString& workDir, const TPortoSettings& portoSettings) - : TChildProcess(exeName, args, env, workDir) - , PortoCtl(portoCtl) - , PortoLayer(portoSettings.Layer) - , MemoryLimit(portoSettings.MemoryLimit) - , ContainerName(WorkDir.substr(WorkDir.rfind("/")+1)) - , InternalWorkDir_("mnt/work") - , InternalExeDir("usr/local/bin") - , TmpDir("TmpDir" + ContainerName) - { - NFs::MakeDirectory(TmpDir); - auto pos = ExeName.rfind("/"); - TString name = ExeName.substr(pos+1); - - if (portoSettings.ContainerNamePrefix) { - ContainerName = portoSettings.ContainerNamePrefix + "/" + ContainerName; - } - - YQL_LOG(DEBUG) << "HardLink " << ExeName << "'" << WorkDir + "/" + name << "'"; - if (NFs::HardLink(ExeName, WorkDir + "/" + name)) { - ExeName = WorkDir + "/" + name; - } else { - YQL_LOG(DEBUG) << "HardLink Failed " << ExeName << "'" << WorkDir + "/" + name << "'"; - } - } - - TString InternalWorkDir() override { - return InternalWorkDir_; - } - - ~TPortoProcess() { - try { - NFs::RemoveRecursive(TmpDir); - } catch (...) { } - } - -private: - void Kill() override { - try { - // see YQL-13760 - TShellCommand cmd(PortoCtl, {"set", ContainerName, "anon_limit", "0"}); - cmd.Run().Wait(); - } catch (...) { - YQL_LOG(DEBUG) << "Cannot set anon_limit: " << CurrentExceptionMessage(); - } - try { - TShellCommand cmd(PortoCtl, {"destroy", ContainerName}); - cmd.Run().Wait(); - } catch (...) { - YQL_LOG(DEBUG) << "Cannot destroy: " << CurrentExceptionMessage(); - } - TChildProcess::Kill(); - } - - void Exec() override { - auto pos = ExeName.rfind("/"); - TString exeDir = ExeName.substr(0, pos); - TString exeName = ExeName.substr(pos+1); - - TString command = InternalExeDir +"/"+ exeName + " "; - for (ui64 i = 1; i < Args.size(); ++i) { - command += Args[i] + " "; - } - - TString caps = "CHOWN;DAC_OVERRIDE;FOWNER;KILL;SETPCAP;IPC_LOCK;SYS_CHROOT;SYS_PTRACE;MKNOD;AUDIT_WRITE;SETFCAP"; - TString env; - for (const auto& [k, v] : Env) { - env += k + "=" + v + ";"; - } - env += "TMPDIR=/tmp;"; - - TString bind; - //bind += WorkDir + " " + InternalWorkDir() + " ro;"; - bind += WorkDir + " " + InternalWorkDir() + " rw;"; // see YQL-11392 - bind += exeDir + " " + InternalExeDir + " ro;"; - bind += TmpDir + " /tmp rw;"; - - TVector<TString> vargs = { - "portoctl", - "exec", - "-L", - PortoLayer.empty() ? "/" : PortoLayer, - ContainerName, - "command=" + command, - "capabilities=" + caps, - "env=" + env, - "bind=" + bind, - "root_readonly=true", - "weak=false", - "cpu_policy=idle", - "anon_limit=" + ToString(MemoryLimit) - }; - - char ** args; - args = new char*[vargs.size() + 1]; - ui32 i; - for (i = 0; i < vargs.size(); ++i) { - args[i] = (char*) vargs[i].c_str(); - } - args[i] = nullptr; - - for (i = 3; i < 32768; ++i) { - close(i); - } - - if (execvp(PortoCtl.c_str(), (char*const*)args) == -1) { - ythrow TSystemError() << "Cannot execl"; - } - } - - const TString PortoCtl; - const TString PortoLayer; - const ui64 MemoryLimit; - TString ContainerName; - - const TString InternalWorkDir_; - const TString InternalExeDir; - const TString TmpDir; -}; - -/*______________________________________________________________________________________________*/ - - -class IPipeTaskRunner: public ITaskRunner { -public: - virtual IOutputStream& GetOutput() = 0; - virtual IInputStream& GetInput() = 0; - [[noreturn]] - virtual void RaiseException() = 0; -}; - -/*______________________________________________________________________________________________*/ - -class TInputChannel : public IInputChannel { -public: - TInputChannel(const ITaskRunner::TPtr& taskRunner, ui64 taskId, ui64 channelId, IInputStream& input, IOutputStream& output) + + virtual void Exec() { + char ** args; + args = new char*[Args.size() + 1]; + ui32 i; + for (i = 0; i < Args.size(); ++i) { + args[i] = (char*) Args[i].c_str(); + } + args[i] = nullptr; + + for (i = 3; i < 32768; ++i) { + close(i); + } + + char** env; + env = new char*[Env.size() + 1]; + i = 0; + for (const auto& [k, v] : Env) { + env[i++] = (char*)(k + "=" + v).c_str(); + } + env[i++] = nullptr; + + if (!WorkDir.empty()) { + NFs::SetCurrentWorkingDirectory(WorkDir); + } + + if (execve(ExeName.c_str(), (char*const*)args, (char*const*)env) == -1) { + ythrow TSystemError() << "Cannot execl"; + } + } +}; + +/*______________________________________________________________________________________________*/ + +struct TProcessHolder { + TProcessHolder() + : Watcher(MakeHolder<TThread>([this] () { Watch(); })) + { + Running.test_and_set(); + Watcher->Start(); + } + + ~TProcessHolder() + { + Running.clear(); + Watcher->Join(); + } + + i64 Size() { + TGuard<TMutex> lock(Mutex); + return Processes.size(); + } + + void Put(const TString& key, THolder<TChildProcess>process) { + TGuard<TMutex> lock(Mutex); + Processes.emplace_back(key, std::move(process)); + } + + THolder<TChildProcess> Acquire(const TString& key, TList<THolder<TChildProcess>>* stopList) { + TGuard<TMutex> lock(Mutex); + THolder<TChildProcess> result; + while (!Processes.empty()) { + auto first = std::move(Processes.front()); + Processes.pop_front(); + if (first.first == key) { + result = std::move(first.second); + break; + } + stopList->push_back(std::move(first.second)); + } + return result; + } + + void Watch() { + while (Running.test()) { + TList<THolder<TChildProcess>> stopList; + { + TGuard<TMutex> lock(Mutex); + auto it = Processes.begin(); + while (it != Processes.end()) { + if (!it->second->IsAlive()) { + YQL_LOG(DEBUG) << "Remove dead process"; + stopList.emplace_back(std::move(it->second)); + it = Processes.erase(it); + } else { + ++it; + } + } + } + + for (const auto& job : stopList) { + job->Kill(); + job->Wait(TDuration::Seconds(1)); + } + Sleep(TDuration::MilliSeconds(1000)); + } + } + + THolder<TThread> Watcher; + std::atomic_flag Running; + + TMutex Mutex; + TList<std::pair<TString, THolder<TChildProcess>>> Processes; +}; + +struct TPortoSettings { + bool Enable; + ui64 MemoryLimit; + TString Layer; + TString ContainerNamePrefix; + + bool operator == (const TPortoSettings& o) const { + return Enable == o.Enable && MemoryLimit == o.MemoryLimit && Layer == o.Layer; + } + + bool operator != (const TPortoSettings& o) const { + return ! operator == (o); + } + + TString ToString() const { + return TStringBuilder() << Enable << "," << MemoryLimit << "," << Layer; + } +}; + +class TPortoProcess: public TChildProcess +{ +public: + TPortoProcess(const TString& portoCtl, const TString& exeName, const TVector<TString>& args, const THashMap<TString, TString>& env, const TString& workDir, const TPortoSettings& portoSettings) + : TChildProcess(exeName, args, env, workDir) + , PortoCtl(portoCtl) + , PortoLayer(portoSettings.Layer) + , MemoryLimit(portoSettings.MemoryLimit) + , ContainerName(WorkDir.substr(WorkDir.rfind("/")+1)) + , InternalWorkDir_("mnt/work") + , InternalExeDir("usr/local/bin") + , TmpDir("TmpDir" + ContainerName) + { + NFs::MakeDirectory(TmpDir); + auto pos = ExeName.rfind("/"); + TString name = ExeName.substr(pos+1); + + if (portoSettings.ContainerNamePrefix) { + ContainerName = portoSettings.ContainerNamePrefix + "/" + ContainerName; + } + + YQL_LOG(DEBUG) << "HardLink " << ExeName << "'" << WorkDir + "/" + name << "'"; + if (NFs::HardLink(ExeName, WorkDir + "/" + name)) { + ExeName = WorkDir + "/" + name; + } else { + YQL_LOG(DEBUG) << "HardLink Failed " << ExeName << "'" << WorkDir + "/" + name << "'"; + } + } + + TString InternalWorkDir() override { + return InternalWorkDir_; + } + + ~TPortoProcess() { + try { + NFs::RemoveRecursive(TmpDir); + } catch (...) { } + } + +private: + void Kill() override { + try { + // see YQL-13760 + TShellCommand cmd(PortoCtl, {"set", ContainerName, "anon_limit", "0"}); + cmd.Run().Wait(); + } catch (...) { + YQL_LOG(DEBUG) << "Cannot set anon_limit: " << CurrentExceptionMessage(); + } + try { + TShellCommand cmd(PortoCtl, {"destroy", ContainerName}); + cmd.Run().Wait(); + } catch (...) { + YQL_LOG(DEBUG) << "Cannot destroy: " << CurrentExceptionMessage(); + } + TChildProcess::Kill(); + } + + void Exec() override { + auto pos = ExeName.rfind("/"); + TString exeDir = ExeName.substr(0, pos); + TString exeName = ExeName.substr(pos+1); + + TString command = InternalExeDir +"/"+ exeName + " "; + for (ui64 i = 1; i < Args.size(); ++i) { + command += Args[i] + " "; + } + + TString caps = "CHOWN;DAC_OVERRIDE;FOWNER;KILL;SETPCAP;IPC_LOCK;SYS_CHROOT;SYS_PTRACE;MKNOD;AUDIT_WRITE;SETFCAP"; + TString env; + for (const auto& [k, v] : Env) { + env += k + "=" + v + ";"; + } + env += "TMPDIR=/tmp;"; + + TString bind; + //bind += WorkDir + " " + InternalWorkDir() + " ro;"; + bind += WorkDir + " " + InternalWorkDir() + " rw;"; // see YQL-11392 + bind += exeDir + " " + InternalExeDir + " ro;"; + bind += TmpDir + " /tmp rw;"; + + TVector<TString> vargs = { + "portoctl", + "exec", + "-L", + PortoLayer.empty() ? "/" : PortoLayer, + ContainerName, + "command=" + command, + "capabilities=" + caps, + "env=" + env, + "bind=" + bind, + "root_readonly=true", + "weak=false", + "cpu_policy=idle", + "anon_limit=" + ToString(MemoryLimit) + }; + + char ** args; + args = new char*[vargs.size() + 1]; + ui32 i; + for (i = 0; i < vargs.size(); ++i) { + args[i] = (char*) vargs[i].c_str(); + } + args[i] = nullptr; + + for (i = 3; i < 32768; ++i) { + close(i); + } + + if (execvp(PortoCtl.c_str(), (char*const*)args) == -1) { + ythrow TSystemError() << "Cannot execl"; + } + } + + const TString PortoCtl; + const TString PortoLayer; + const ui64 MemoryLimit; + TString ContainerName; + + const TString InternalWorkDir_; + const TString InternalExeDir; + const TString TmpDir; +}; + +/*______________________________________________________________________________________________*/ + + +class IPipeTaskRunner: public ITaskRunner { +public: + virtual IOutputStream& GetOutput() = 0; + virtual IInputStream& GetInput() = 0; + [[noreturn]] + virtual void RaiseException() = 0; +}; + +/*______________________________________________________________________________________________*/ + +class TInputChannel : public IInputChannel { +public: + TInputChannel(const ITaskRunner::TPtr& taskRunner, ui64 taskId, ui64 channelId, IInputStream& input, IOutputStream& output) : TaskId(taskId) - , ChannelId(channelId) - , Input(input) - , Output(output) + , ChannelId(channelId) + , Input(input) + , Output(output) , ProtocolVersion(taskRunner->GetProtocolVersion()) - { } - - i64 GetFreeSpace() override { - if (ProtocolVersion <= 1) { - return std::numeric_limits<i64>::max(); - } - - NDqProto::TCommandHeader header; - header.SetVersion(2); - header.SetCommand(NDqProto::TCommandHeader::GET_FREE_SPACE); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TGetFreeSpaceResponse response; - response.Load(&Input); - return response.GetFreeSpace(); - } - - void Push(NDqProto::TData&& data) override { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::PUSH); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - data.Save(&Output); - } - - void Finish() override { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::FINISH); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - } - -private: - ui64 TaskId; - ui64 ChannelId; - - IInputStream& Input; - IOutputStream& Output; - - TString SerializedInputType; - - i32 ProtocolVersion; -}; - -class TDqInputChannel: public IDqInputChannel { -public: - TDqInputChannel(const IInputChannel::TPtr& channel, ui64 taskId, ui64 channelId, IPipeTaskRunner* taskRunner) - : Delegate(channel) - , TaskId(taskId) - , ChannelId(channelId) - , Stats(ChannelId) - , TaskRunner(taskRunner) - , Input(TaskRunner->GetInput()) - , Output(TaskRunner->GetOutput()) - { } - - ui64 GetChannelId() const override { - return ChannelId; - } - - i64 GetFreeSpace() const override { - try { - return Delegate->GetFreeSpace(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - ui64 GetStoredBytes() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::GET_STORED_BYTES); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TGetStoredBytesResponse response; - response.Load(&Input); - - return response.GetResult(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - + { } + + i64 GetFreeSpace() override { + if (ProtocolVersion <= 1) { + return std::numeric_limits<i64>::max(); + } + + NDqProto::TCommandHeader header; + header.SetVersion(2); + header.SetCommand(NDqProto::TCommandHeader::GET_FREE_SPACE); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TGetFreeSpaceResponse response; + response.Load(&Input); + return response.GetFreeSpace(); + } + + void Push(NDqProto::TData&& data) override { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::PUSH); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + data.Save(&Output); + } + + void Finish() override { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::FINISH); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + } + +private: + ui64 TaskId; + ui64 ChannelId; + + IInputStream& Input; + IOutputStream& Output; + + TString SerializedInputType; + + i32 ProtocolVersion; +}; + +class TDqInputChannel: public IDqInputChannel { +public: + TDqInputChannel(const IInputChannel::TPtr& channel, ui64 taskId, ui64 channelId, IPipeTaskRunner* taskRunner) + : Delegate(channel) + , TaskId(taskId) + , ChannelId(channelId) + , Stats(ChannelId) + , TaskRunner(taskRunner) + , Input(TaskRunner->GetInput()) + , Output(TaskRunner->GetOutput()) + { } + + ui64 GetChannelId() const override { + return ChannelId; + } + + i64 GetFreeSpace() const override { + try { + return Delegate->GetFreeSpace(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + ui64 GetStoredBytes() const override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::GET_STORED_BYTES); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TGetStoredBytesResponse response; + response.Load(&Input); + + return response.GetResult(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + bool Empty() const override { ythrow yexception() << "unimplemented"; } - void Push(NDqProto::TData&& data) override { - try { - return Delegate->Push(std::move(data)); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - [[nodiscard]] - bool Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch) override { - Y_UNUSED(batch); - ythrow yexception() << "unimplemented"; - } - - void Finish() override { - try { - Delegate->Finish(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - bool IsFinished() const override { - ythrow yexception() << "unimplemented"; - } - + void Push(NDqProto::TData&& data) override { + try { + return Delegate->Push(std::move(data)); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + [[nodiscard]] + bool Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch) override { + Y_UNUSED(batch); + ythrow yexception() << "unimplemented"; + } + + void Finish() override { + try { + Delegate->Finish(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + bool IsFinished() const override { + ythrow yexception() << "unimplemented"; + } + const TDqInputChannelStats* GetStats() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::GET_STATS_INPUT); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TGetStatsInputResponse response; - response.Load(&Input); - - FromProto(&Stats, response.GetStats()); - return &Stats; - } catch (...) { - TaskRunner->RaiseException(); - } - } - - NKikimr::NMiniKQL::TType* GetInputType() const override { - ythrow yexception() << "unimplemented"; - } - + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::GET_STATS_INPUT); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TGetStatsInputResponse response; + response.Load(&Input); + + FromProto(&Stats, response.GetStats()); + return &Stats; + } catch (...) { + TaskRunner->RaiseException(); + } + } + + NKikimr::NMiniKQL::TType* GetInputType() const override { + ythrow yexception() << "unimplemented"; + } + void Pause() override { Y_FAIL("Checkpoints are not supported"); } @@ -681,18 +681,18 @@ public: return false; } -private: - IInputChannel::TPtr Delegate; - ui64 TaskId; - ui64 ChannelId; - mutable TDqInputChannelStats Stats; - - IPipeTaskRunner* TaskRunner; - IInputStream& Input; - IOutputStream& Output; -}; - -class TDqSource: public IStringSource { +private: + IInputChannel::TPtr Delegate; + ui64 TaskId; + ui64 ChannelId; + mutable TDqInputChannelStats Stats; + + IPipeTaskRunner* TaskRunner; + IInputStream& Input; + IOutputStream& Output; +}; + +class TDqSource: public IStringSource { public: TDqSource(ui64 taskId, ui64 inputIndex, IPipeTaskRunner* taskRunner) : TaskId(taskId) @@ -734,41 +734,41 @@ public: ythrow yexception() << "unimplemented"; } - void PushString(TVector<TString>&& batch, i64 space) override { - if (space > static_cast<i64>(256_MB)) { - throw yexception() << "Too big batch " << space; - } - NDqProto::TSourcePushRequest data; - data.SetSpace(space); - data.SetChunks(batch.size()); - - NDqProto::TCommandHeader header; - header.SetVersion(4); - header.SetCommand(NDqProto::TCommandHeader::PUSH_SOURCE); - header.SetTaskId(TaskId); - header.SetChannelId(InputIndex); - header.Save(&Output); - data.Save(&Output); - - i64 partSize = 32_MB; - for (auto& b : batch) { - NDqProto::TSourcePushChunk chunk; - i64 parts = (b.size() + partSize - 1) / partSize; - chunk.SetParts(parts); - if (parts == 1) { - chunk.SetString(std::move(b)); - chunk.Save(&Output); - } else { - chunk.Save(&Output); - for (i64 i = 0; i < parts; i++) { - NDqProto::TSourcePushPart part; - part.SetString(b.substr(i*partSize, partSize)); - part.Save(&Output); - } - } - } - } - + void PushString(TVector<TString>&& batch, i64 space) override { + if (space > static_cast<i64>(256_MB)) { + throw yexception() << "Too big batch " << space; + } + NDqProto::TSourcePushRequest data; + data.SetSpace(space); + data.SetChunks(batch.size()); + + NDqProto::TCommandHeader header; + header.SetVersion(4); + header.SetCommand(NDqProto::TCommandHeader::PUSH_SOURCE); + header.SetTaskId(TaskId); + header.SetChannelId(InputIndex); + header.Save(&Output); + data.Save(&Output); + + i64 partSize = 32_MB; + for (auto& b : batch) { + NDqProto::TSourcePushChunk chunk; + i64 parts = (b.size() + partSize - 1) / partSize; + chunk.SetParts(parts); + if (parts == 1) { + chunk.SetString(std::move(b)); + chunk.Save(&Output); + } else { + chunk.Save(&Output); + for (i64 i = 0; i < parts; i++) { + NDqProto::TSourcePushPart part; + part.SetString(b.substr(i*partSize, partSize)); + part.Save(&Output); + } + } + } + } + void Push(NKikimr::NMiniKQL::TUnboxedValueVector&& batch, i64 space) override { auto inputType = GetInputType(); NDqProto::TSourcePushRequest data; @@ -869,521 +869,521 @@ private: mutable NKikimr::NMiniKQL::TType* InputType = nullptr; }; -/*______________________________________________________________________________________________*/ - -class TOutputChannel : public IOutputChannel { -public: - TOutputChannel(ui64 taskId, ui64 channelId, IInputStream& input, IOutputStream& output) - : TaskId(taskId) - , ChannelId(channelId) - , Input(input) - , Output(output) - { } - - [[nodiscard]] +/*______________________________________________________________________________________________*/ + +class TOutputChannel : public IOutputChannel { +public: + TOutputChannel(ui64 taskId, ui64 channelId, IInputStream& input, IOutputStream& output) + : TaskId(taskId) + , ChannelId(channelId) + , Input(input) + , Output(output) + { } + + [[nodiscard]] NDqProto::TPopResponse Pop(NDqProto::TData& data, ui64) override { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::POP); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TPopResponse response; - response.Load(&Input); - data = std::move(*response.MutableData()); - return response; - } - + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::POP); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TPopResponse response; + response.Load(&Input); + data = std::move(*response.MutableData()); + return response; + } + bool IsFinished() const override { - if (Finished) { - return true; - } - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::IS_FINISHED); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TIsFinishedResponse response; - response.Load(&Input); - Finished = response.GetResult(); - return Finished; - } - -private: - mutable bool Finished = false; - ui64 TaskId; - ui64 ChannelId; - - IInputStream& Input; - IOutputStream& Output; -}; - -class TDqOutputChannel: public IDqOutputChannel { -public: - TDqOutputChannel(const IOutputChannel::TPtr& channel, ui64 taskId, ui64 channelId, IPipeTaskRunner* taskRunner) - : Delegate(channel) - , TaskId(taskId) - , ChannelId(channelId) - , Stats(ChannelId) - , TaskRunner(taskRunner) - , Input(TaskRunner->GetInput()) - , Output(TaskRunner->GetOutput()) - { - Y_UNUSED(Input); - } - - ui64 GetChannelId() const override { - return ChannelId; - } - - ui64 GetValuesCount(bool inMemoryOnly) const override { - Y_UNUSED(inMemoryOnly); - ythrow yexception() << "unimplemented"; - } - - // <| producer methods - [[nodiscard]] - bool IsFull() const override { - ythrow yexception() << "unimplemented"; - }; - + if (Finished) { + return true; + } + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::IS_FINISHED); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TIsFinishedResponse response; + response.Load(&Input); + Finished = response.GetResult(); + return Finished; + } + +private: + mutable bool Finished = false; + ui64 TaskId; + ui64 ChannelId; + + IInputStream& Input; + IOutputStream& Output; +}; + +class TDqOutputChannel: public IDqOutputChannel { +public: + TDqOutputChannel(const IOutputChannel::TPtr& channel, ui64 taskId, ui64 channelId, IPipeTaskRunner* taskRunner) + : Delegate(channel) + , TaskId(taskId) + , ChannelId(channelId) + , Stats(ChannelId) + , TaskRunner(taskRunner) + , Input(TaskRunner->GetInput()) + , Output(TaskRunner->GetOutput()) + { + Y_UNUSED(Input); + } + + ui64 GetChannelId() const override { + return ChannelId; + } + + ui64 GetValuesCount(bool inMemoryOnly) const override { + Y_UNUSED(inMemoryOnly); + ythrow yexception() << "unimplemented"; + } + + // <| producer methods + [[nodiscard]] + bool IsFull() const override { + ythrow yexception() << "unimplemented"; + }; + // can throw TDqChannelStorageException - void Push(NUdf::TUnboxedValue&& value) override { - Y_UNUSED(value); - ythrow yexception() << "unimplemented"; - } - + void Push(NUdf::TUnboxedValue&& value) override { + Y_UNUSED(value); + ythrow yexception() << "unimplemented"; + } + void Push(NDqProto::TCheckpoint&& checkpoint) override { Y_UNUSED(checkpoint); ythrow yexception() << "unimplemented"; } - void Finish() override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::FINISH_OUTPUT); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - } catch (...) { - TaskRunner->RaiseException(); - } - } - // |> - - // <| consumer methods + void Finish() override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::FINISH_OUTPUT); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + } catch (...) { + TaskRunner->RaiseException(); + } + } + // |> + + // <| consumer methods bool HasData() const override { ythrow yexception() << "unimplemented"; } - bool IsFinished() const override { - try { - return Delegate->IsFinished(); - } catch (...) { - TaskRunner->RaiseException(); - } - } + bool IsFinished() const override { + try { + return Delegate->IsFinished(); + } catch (...) { + TaskRunner->RaiseException(); + } + } // can throw TDqChannelStorageException - [[nodiscard]] - bool Pop(NDqProto::TData& data, ui64 bytes) override { - try { - auto response = Delegate->Pop(data, bytes); - return response.GetResult(); - } catch (...) { - TaskRunner->RaiseException(); - } - } + [[nodiscard]] + bool Pop(NDqProto::TData& data, ui64 bytes) override { + try { + auto response = Delegate->Pop(data, bytes); + return response.GetResult(); + } catch (...) { + TaskRunner->RaiseException(); + } + } bool Pop(NDqProto::TCheckpoint&) override { - return false; + return false; } - // Only for data-queries - // TODO: remove this method and create independent Data- and Stream-query implementations. - // Stream-query implementation should be without PopAll method. - // Data-query implementation should be one-shot for Pop (a-la PopAll) call and without ChannelStorage. + // Only for data-queries + // TODO: remove this method and create independent Data- and Stream-query implementations. + // Stream-query implementation should be without PopAll method. + // Data-query implementation should be one-shot for Pop (a-la PopAll) call and without ChannelStorage. // can throw TDqChannelStorageException - [[nodiscard]] - bool PopAll(NDqProto::TData& data) override { - Y_UNUSED(data); - ythrow yexception() << "unimplemented"; - } + [[nodiscard]] + bool PopAll(NDqProto::TData& data) override { + Y_UNUSED(data); + ythrow yexception() << "unimplemented"; + } bool PopAll(NKikimr::NMiniKQL::TUnboxedValueVector& rows) override { Y_UNUSED(rows); ythrow yexception() << "unimplemented"; } - // |> - - ui64 Drop() override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::DROP_OUTPUT); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TDropOutputResponse response; - response.Load(&Input); - - return response.GetResult(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - NKikimr::NMiniKQL::TType* GetOutputType() const override { - Y_FAIL("Unimplemented"); - return nullptr; - } - + // |> + + ui64 Drop() override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::DROP_OUTPUT); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TDropOutputResponse response; + response.Load(&Input); + + return response.GetResult(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + NKikimr::NMiniKQL::TType* GetOutputType() const override { + Y_FAIL("Unimplemented"); + return nullptr; + } + const TDqOutputChannelStats* GetStats() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::GET_STATS_OUTPUT); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - - NDqProto::TGetStatsOutputResponse response; - response.Load(&Input); - - FromProto(&Stats, response.GetStats()); - return &Stats; - } catch (...) { - TaskRunner->RaiseException(); - } - } - - void Terminate() override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::TERMINATE_OUTPUT); - header.SetTaskId(TaskId); - header.SetChannelId(ChannelId); - header.Save(&Output); - } catch (...) { - TaskRunner->RaiseException(); - } - } - -private: - IOutputChannel::TPtr Delegate; - ui64 TaskId; - ui64 ChannelId; - mutable TDqOutputChannelStats Stats; - IPipeTaskRunner* TaskRunner; - IInputStream& Input; - IOutputStream& Output; -}; - -class TDqSink : public IStringSink { -public: - TDqSink(ui64 taskId, ui64 index, IPipeTaskRunner* taskRunner) - : TaskId(taskId) - , Index(index) - , TaskRunner(taskRunner) - , Input(TaskRunner->GetInput()) - , Output(TaskRunner->GetOutput()) - , Stats(Index) - { } - - ui64 GetOutputIndex() const override { - return Index; - } - - ui64 PopString(TVector<TString>& batch, ui64 bytes) override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(5); - header.SetCommand(NDqProto::TCommandHeader::SINK_POP); - header.SetTaskId(TaskId); - header.SetChannelId(Index); - header.Save(&Output); - - NDqProto::TSinkPopRequest request; - request.SetBytes(bytes); - request.SetRaw(true); - header.Save(&Output); - - NDqProto::TSinkPopResponse response; - response.Load(&Input); - - for (auto& row : response.GetString()) { - batch.emplace_back(std::move(row)); - } - - return response.GetBytes(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - ui64 Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch, ui64 bytes) override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(5); - header.SetCommand(NDqProto::TCommandHeader::SINK_POP); - header.SetTaskId(TaskId); - header.SetChannelId(Index); - header.Save(&Output); - - NDqProto::TSinkPopRequest request; - request.SetBytes(bytes); - header.Save(&Output); - - NDqProto::TSinkPopResponse response; - response.Load(&Input); - - TDqDataSerializer dataSerializer( - TaskRunner->GetTypeEnv(), - TaskRunner->GetHolderFactory(), - NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); - - dataSerializer.Deserialize( - response.GetData(), - GetOutputType(), - batch); - - return response.GetBytes(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - bool IsFinished() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(5); - header.SetCommand(NDqProto::TCommandHeader::SINK_IS_FINISHED); - header.SetTaskId(TaskId); - header.SetChannelId(Index); - header.Save(&Output); - - NDqProto::TIsFinishedResponse response; - response.Load(&Input); - - return response.GetResult(); - } catch (...) { - TaskRunner->RaiseException(); - } - } - - NKikimr::NMiniKQL::TType* GetOutputType() const override { - if (OutputType) { - return OutputType; - } - - NDqProto::TCommandHeader header; - header.SetVersion(5); - header.SetCommand(NDqProto::TCommandHeader::SINK_OUTPUT_TYPE); - header.SetTaskId(TaskId); - header.SetChannelId(Index); - header.Save(&Output); - - NDqProto::TGetTypeResponse response; - response.Load(&Input); - - OutputType = static_cast<NKikimr::NMiniKQL::TType*>( - NKikimr::NMiniKQL::DeserializeNode( - response.GetResult(), - TaskRunner->GetTypeEnv())); - return OutputType; - } - - const TDqSinkStats* GetStats() const override { - try { - NDqProto::TCommandHeader header; - header.SetVersion(5); - header.SetCommand(NDqProto::TCommandHeader::SINK_STATS); - header.SetTaskId(TaskId); - header.SetChannelId(Index); - header.Save(&Output); - - NDqProto::TSinkStatsResponse response; - response.Load(&Input); - - FromProto(&Stats, response.GetStats()); - return &Stats; - } catch (...) { - TaskRunner->RaiseException(); - } - } - - void Finish() override { - Y_FAIL("Unimplemented"); - } - - bool Pop(NDqProto::TCheckpoint& checkpoint) override { - Y_UNUSED(checkpoint); - Y_FAIL("Checkpoints are not supported"); - } - - bool IsFull() const override { - Y_FAIL("Unimplemented"); - } - - void Push(NUdf::TUnboxedValue&& value) override { - Y_UNUSED(value); - Y_FAIL("Unimplemented"); - } - - void Push(NDqProto::TCheckpoint&& checkpoint) override { - Y_UNUSED(checkpoint); - Y_FAIL("Checkpoints are not supported"); - } - - bool HasData() const override { - Y_FAIL("Unimplemented"); - } - -private: - ui64 TaskId; - ui64 Index; - IPipeTaskRunner* TaskRunner; - - IInputStream& Input; - IOutputStream& Output; - - mutable TDqSinkStats Stats; - mutable NKikimr::NMiniKQL::TType* OutputType = nullptr; -}; - -/*______________________________________________________________________________________________*/ - -class TTaskRunner: public IPipeTaskRunner { -public: - TTaskRunner( - const NDqProto::TDqTask& task, - THolder<TChildProcess>&& command, - ui64 stageId, - const TString& traceId) - : TraceId(traceId) - , Task(task) - , Alloc(NKikimr::TAlignedPagePoolCounters(), true) - , TypeEnv(Alloc) - , MemInfo("TDqTaskRunnerProxy") - , HolderFactory(Alloc.Ref(), MemInfo) - , Running(true) - , Command(std::move(command)) - , StderrReader(MakeHolder<TThread>([this] () { ReadStderr(); })) - , Output(Command->GetStdin()) - , Input(Command->GetStdout()) - , TaskId(Task.GetId()) - , StageId(stageId) - { - Alloc.Release(); - StderrReader->Start(); + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::GET_STATS_OUTPUT); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + + NDqProto::TGetStatsOutputResponse response; + response.Load(&Input); + + FromProto(&Stats, response.GetStats()); + return &Stats; + } catch (...) { + TaskRunner->RaiseException(); + } + } + + void Terminate() override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::TERMINATE_OUTPUT); + header.SetTaskId(TaskId); + header.SetChannelId(ChannelId); + header.Save(&Output); + } catch (...) { + TaskRunner->RaiseException(); + } + } + +private: + IOutputChannel::TPtr Delegate; + ui64 TaskId; + ui64 ChannelId; + mutable TDqOutputChannelStats Stats; + IPipeTaskRunner* TaskRunner; + IInputStream& Input; + IOutputStream& Output; +}; + +class TDqSink : public IStringSink { +public: + TDqSink(ui64 taskId, ui64 index, IPipeTaskRunner* taskRunner) + : TaskId(taskId) + , Index(index) + , TaskRunner(taskRunner) + , Input(TaskRunner->GetInput()) + , Output(TaskRunner->GetOutput()) + , Stats(Index) + { } + + ui64 GetOutputIndex() const override { + return Index; + } + + ui64 PopString(TVector<TString>& batch, ui64 bytes) override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(5); + header.SetCommand(NDqProto::TCommandHeader::SINK_POP); + header.SetTaskId(TaskId); + header.SetChannelId(Index); + header.Save(&Output); + + NDqProto::TSinkPopRequest request; + request.SetBytes(bytes); + request.SetRaw(true); + header.Save(&Output); + + NDqProto::TSinkPopResponse response; + response.Load(&Input); + + for (auto& row : response.GetString()) { + batch.emplace_back(std::move(row)); + } + + return response.GetBytes(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + ui64 Pop(NKikimr::NMiniKQL::TUnboxedValueVector& batch, ui64 bytes) override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(5); + header.SetCommand(NDqProto::TCommandHeader::SINK_POP); + header.SetTaskId(TaskId); + header.SetChannelId(Index); + header.Save(&Output); + + NDqProto::TSinkPopRequest request; + request.SetBytes(bytes); + header.Save(&Output); + + NDqProto::TSinkPopResponse response; + response.Load(&Input); + + TDqDataSerializer dataSerializer( + TaskRunner->GetTypeEnv(), + TaskRunner->GetHolderFactory(), + NDqProto::DATA_TRANSPORT_UV_PICKLE_1_0); + + dataSerializer.Deserialize( + response.GetData(), + GetOutputType(), + batch); + + return response.GetBytes(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + bool IsFinished() const override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(5); + header.SetCommand(NDqProto::TCommandHeader::SINK_IS_FINISHED); + header.SetTaskId(TaskId); + header.SetChannelId(Index); + header.Save(&Output); + + NDqProto::TIsFinishedResponse response; + response.Load(&Input); + + return response.GetResult(); + } catch (...) { + TaskRunner->RaiseException(); + } + } + + NKikimr::NMiniKQL::TType* GetOutputType() const override { + if (OutputType) { + return OutputType; + } + + NDqProto::TCommandHeader header; + header.SetVersion(5); + header.SetCommand(NDqProto::TCommandHeader::SINK_OUTPUT_TYPE); + header.SetTaskId(TaskId); + header.SetChannelId(Index); + header.Save(&Output); + + NDqProto::TGetTypeResponse response; + response.Load(&Input); + + OutputType = static_cast<NKikimr::NMiniKQL::TType*>( + NKikimr::NMiniKQL::DeserializeNode( + response.GetResult(), + TaskRunner->GetTypeEnv())); + return OutputType; + } + + const TDqSinkStats* GetStats() const override { + try { + NDqProto::TCommandHeader header; + header.SetVersion(5); + header.SetCommand(NDqProto::TCommandHeader::SINK_STATS); + header.SetTaskId(TaskId); + header.SetChannelId(Index); + header.Save(&Output); + + NDqProto::TSinkStatsResponse response; + response.Load(&Input); + + FromProto(&Stats, response.GetStats()); + return &Stats; + } catch (...) { + TaskRunner->RaiseException(); + } + } + + void Finish() override { + Y_FAIL("Unimplemented"); + } + + bool Pop(NDqProto::TCheckpoint& checkpoint) override { + Y_UNUSED(checkpoint); + Y_FAIL("Checkpoints are not supported"); + } + + bool IsFull() const override { + Y_FAIL("Unimplemented"); + } + + void Push(NUdf::TUnboxedValue&& value) override { + Y_UNUSED(value); + Y_FAIL("Unimplemented"); + } + + void Push(NDqProto::TCheckpoint&& checkpoint) override { + Y_UNUSED(checkpoint); + Y_FAIL("Checkpoints are not supported"); + } + + bool HasData() const override { + Y_FAIL("Unimplemented"); + } + +private: + ui64 TaskId; + ui64 Index; + IPipeTaskRunner* TaskRunner; + + IInputStream& Input; + IOutputStream& Output; + + mutable TDqSinkStats Stats; + mutable NKikimr::NMiniKQL::TType* OutputType = nullptr; +}; + +/*______________________________________________________________________________________________*/ + +class TTaskRunner: public IPipeTaskRunner { +public: + TTaskRunner( + const NDqProto::TDqTask& task, + THolder<TChildProcess>&& command, + ui64 stageId, + const TString& traceId) + : TraceId(traceId) + , Task(task) + , Alloc(NKikimr::TAlignedPagePoolCounters(), true) + , TypeEnv(Alloc) + , MemInfo("TDqTaskRunnerProxy") + , HolderFactory(Alloc.Ref(), MemInfo) + , Running(true) + , Command(std::move(command)) + , StderrReader(MakeHolder<TThread>([this] () { ReadStderr(); })) + , Output(Command->GetStdin()) + , Input(Command->GetStdout()) + , TaskId(Task.GetId()) + , StageId(stageId) + { + Alloc.Release(); + StderrReader->Start(); InitTaskMeta(); - } - + } + ~TTaskRunner() { - Alloc.Acquire(); - Command->Kill(); - Command->Wait(TDuration::Seconds(0)); - } - + Alloc.Acquire(); + Command->Kill(); + Command->Wait(TDuration::Seconds(0)); + } + void ReadStderr() { - auto& input = Command->GetStderr(); - char buf[1024]; - size_t size; - while ((size = input.Read(buf, sizeof(buf))) > 0) { - auto str = TString(buf, size); - Stderr += str; - YQL_LOG(DEBUG) << "stderr (" << StageId << " " << TraceId << " ) > `" << str << "'"; - } - YQL_LOG(DEBUG) << "stderr (" << StageId << " " << TraceId << " ) finished"; - } - + auto& input = Command->GetStderr(); + char buf[1024]; + size_t size; + while ((size = input.Read(buf, sizeof(buf))) > 0) { + auto str = TString(buf, size); + Stderr += str; + YQL_LOG(DEBUG) << "stderr (" << StageId << " " << TraceId << " ) > `" << str << "'"; + } + YQL_LOG(DEBUG) << "stderr (" << StageId << " " << TraceId << " ) finished"; + } + ui64 GetTaskId() const override { - return TaskId; - } - - i32 GetProtocolVersion() override { - if (ProtocolVersion < 0) { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::VERSION); - header.Save(&Output); - - NDqProto::TGetVersionResponse response; - response.Load(&Input); - - ProtocolVersion = response.GetVersion(); - } - - return ProtocolVersion; - } - - NYql::NDqProto::TPrepareResponse Prepare() override { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::PREPARE); - header.Save(&Output); - - NDqProto::TPrepareRequest request; - *request.MutableTask() = Task; - request.Save(&Output); - - NYql::NDqProto::TPrepareResponse ret; - ret.Load(&Input); - return ret; - } - - NYql::NDqProto::TRunResponse Run() override { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::RUN); - header.SetTaskId(Task.GetId()); - header.Save(&Output); - - NDqProto::TRunResponse response; - response.Load(&Input); - return response; - } - + return TaskId; + } + + i32 GetProtocolVersion() override { + if (ProtocolVersion < 0) { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::VERSION); + header.Save(&Output); + + NDqProto::TGetVersionResponse response; + response.Load(&Input); + + ProtocolVersion = response.GetVersion(); + } + + return ProtocolVersion; + } + + NYql::NDqProto::TPrepareResponse Prepare() override { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::PREPARE); + header.Save(&Output); + + NDqProto::TPrepareRequest request; + *request.MutableTask() = Task; + request.Save(&Output); + + NYql::NDqProto::TPrepareResponse ret; + ret.Load(&Input); + return ret; + } + + NYql::NDqProto::TRunResponse Run() override { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::RUN); + header.SetTaskId(Task.GetId()); + header.Save(&Output); + + NDqProto::TRunResponse response; + response.Load(&Input); + return response; + } + IInputChannel::TPtr GetInputChannel(ui64 channelId) override { - return new TInputChannel(this, Task.GetId(), channelId, Input, Output); + return new TInputChannel(this, Task.GetId(), channelId, Input, Output); } IOutputChannel::TPtr GetOutputChannel(ui64 channelId) override { - return new TOutputChannel(Task.GetId(), channelId, Input, Output); + return new TOutputChannel(Task.GetId(), channelId, Input, Output); + } + + IDqSource::TPtr GetSource(ui64 index) override { + return new TDqSource(Task.GetId(), index, this); + } + + TDqSink::TPtr GetSink(ui64 index) override { + return new TDqSink(Task.GetId(), index, this); } - IDqSource::TPtr GetSource(ui64 index) override { - return new TDqSource(Task.GetId(), index, this); - } - - TDqSink::TPtr GetSink(ui64 index) override { - return new TDqSink(Task.GetId(), index, this); - } - const NMiniKQL::TTypeEnvironment& GetTypeEnv() const override { - return TypeEnv; - } - + return TypeEnv; + } + const NMiniKQL::THolderFactory& GetHolderFactory() const override { - return HolderFactory; + return HolderFactory; } - const THashMap<TString, TString>& GetSecureParams() const override { + const THashMap<TString, TString>& GetSecureParams() const override { return SecureParams; } - const THashMap<TString, TString>& GetTaskParams() const override { + const THashMap<TString, TString>& GetTaskParams() const override { return TaskParams; } TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override { - auto guard = TypeEnv.BindAllocator(); + auto guard = TypeEnv.BindAllocator(); if (memoryLimit) { guard.GetMutex()->SetLimit(*memoryLimit); } @@ -1391,55 +1391,55 @@ public: } bool IsAllocatorAttached() override { - return TypeEnv.GetAllocator().IsAttached(); - } - - void Kill() override { - bool expected = true; - if (!Running.compare_exchange_strong(expected, false)) { - return; - } - - Command->Kill(); - Command->Wait(TDuration::Seconds(0)); - StderrReader->Join(); - } - - TStatus GetStatus() override { - bool expected = true; - if (!Running.compare_exchange_strong(expected, false)) { - return {Code, Stderr}; - } - - try { - NDqProto::TCommandHeader header; - header.SetVersion(1); - header.SetCommand(NDqProto::TCommandHeader::STOP); - header.Save(&Output); - } catch (...) { } - Code = Command->Wait(); - StderrReader->Join(); - return {Code, Stderr}; - } - - [[noreturn]] + return TypeEnv.GetAllocator().IsAttached(); + } + + void Kill() override { + bool expected = true; + if (!Running.compare_exchange_strong(expected, false)) { + return; + } + + Command->Kill(); + Command->Wait(TDuration::Seconds(0)); + StderrReader->Join(); + } + + TStatus GetStatus() override { + bool expected = true; + if (!Running.compare_exchange_strong(expected, false)) { + return {Code, Stderr}; + } + + try { + NDqProto::TCommandHeader header; + header.SetVersion(1); + header.SetCommand(NDqProto::TCommandHeader::STOP); + header.Save(&Output); + } catch (...) { } + Code = Command->Wait(); + StderrReader->Join(); + return {Code, Stderr}; + } + + [[noreturn]] void RaiseException() override { - auto status = GetStatus(); - TString message; - message += "ExitCode: " + ToString(status.ExitCode) + "\n"; - message += "Stderr:\n" + status.Stderr + "\n"; - ythrow yexception() << message; - } - + auto status = GetStatus(); + TString message; + message += "ExitCode: " + ToString(status.ExitCode) + "\n"; + message += "Stderr:\n" + status.Stderr + "\n"; + ythrow yexception() << message; + } + IOutputStream& GetOutput() override { - return Output; - } - + return Output; + } + IInputStream& GetInput() override { - return Input; - } - -private: + return Input; + } + +private: void InitTaskMeta() { Yql::DqsProto::TTaskMeta taskMeta; Task.GetMeta().UnpackTo(&taskMeta); @@ -1454,87 +1454,87 @@ private: } private: - const TString TraceId; - NDqProto::TDqTask Task; + const TString TraceId; + NDqProto::TDqTask Task; THashMap<TString, TString> SecureParams; THashMap<TString, TString> TaskParams; - - NKikimr::NMiniKQL::TScopedAlloc Alloc; - NKikimr::NMiniKQL::TTypeEnvironment TypeEnv; - NKikimr::NMiniKQL::TMemoryUsageInfo MemInfo; - NKikimr::NMiniKQL::THolderFactory HolderFactory; - - std::atomic<bool> Running; + + NKikimr::NMiniKQL::TScopedAlloc Alloc; + NKikimr::NMiniKQL::TTypeEnvironment TypeEnv; + NKikimr::NMiniKQL::TMemoryUsageInfo MemInfo; + NKikimr::NMiniKQL::THolderFactory HolderFactory; + + std::atomic<bool> Running; int Code = -1; - TString Stderr; - THolder<TChildProcess> Command; - THolder<TThread> StderrReader; - IOutputStream& Output; - IInputStream& Input; - - i32 ProtocolVersion = -1; - const ui64 TaskId; - const ui64 StageId; -}; - -class TDqTaskRunner: public NDq::IDqTaskRunner { -public: - TDqTaskRunner( - const NDqProto::TDqTask& task, - THolder<TChildProcess>&& command, - ui64 stageId, - const TString& traceId) - : Delegate(new TTaskRunner(task, std::move(command), stageId, traceId)) - , Task(task) - { } - + TString Stderr; + THolder<TChildProcess> Command; + THolder<TThread> StderrReader; + IOutputStream& Output; + IInputStream& Input; + + i32 ProtocolVersion = -1; + const ui64 TaskId; + const ui64 StageId; +}; + +class TDqTaskRunner: public NDq::IDqTaskRunner { +public: + TDqTaskRunner( + const NDqProto::TDqTask& task, + THolder<TChildProcess>&& command, + ui64 stageId, + const TString& traceId) + : Delegate(new TTaskRunner(task, std::move(command), stageId, traceId)) + , Task(task) + { } + ui64 GetTaskId() const override { return Task.GetId(); - } - + } + void Prepare(const NDqProto::TDqTask& task, const TDqTaskRunnerMemoryLimits& memoryLimits, const IDqTaskRunnerExecutionContext& execCtx, const TDqTaskRunnerParameterProvider&) override { Y_UNUSED(memoryLimits); Y_UNUSED(execCtx); Y_VERIFY(Task.GetId() == task.GetId()); - try { - auto result = Delegate->Prepare(); - Y_UNUSED(result); - } catch (...) { - Delegate->RaiseException(); - } - } - - ERunStatus Run() override - { - try { - auto response = Delegate->Run(); - return static_cast<NDq::ERunStatus>(response.GetResult()); - } catch (...) { - Delegate->RaiseException(); - } - } - - bool HasEffects() const override - { - return false; - } - - IDqInputChannel::TPtr GetInputChannel(ui64 channelId) override - { - auto& channel = InputChannels[channelId]; - if (!channel) { - channel = new TDqInputChannel( - Delegate->GetInputChannel(channelId), - Task.GetId(), - channelId, - Delegate.Get()); - Stats.InputChannels[channelId] = channel->GetStats(); - } - return channel; - } - + try { + auto result = Delegate->Prepare(); + Y_UNUSED(result); + } catch (...) { + Delegate->RaiseException(); + } + } + + ERunStatus Run() override + { + try { + auto response = Delegate->Run(); + return static_cast<NDq::ERunStatus>(response.GetResult()); + } catch (...) { + Delegate->RaiseException(); + } + } + + bool HasEffects() const override + { + return false; + } + + IDqInputChannel::TPtr GetInputChannel(ui64 channelId) override + { + auto& channel = InputChannels[channelId]; + if (!channel) { + channel = new TDqInputChannel( + Delegate->GetInputChannel(channelId), + Task.GetId(), + channelId, + Delegate.Get()); + Stats.InputChannels[channelId] = channel->GetStats(); + } + return channel; + } + IDqSource::TPtr GetSource(ui64 inputIndex) override { auto& source = Sources[inputIndex]; if (!source) { @@ -1547,36 +1547,36 @@ public: return source; } - IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) override - { - auto& channel = OutputChannels[channelId]; - if (!channel) { - channel = new TDqOutputChannel( - Delegate->GetOutputChannel(channelId), - Task.GetId(), - channelId, - Delegate.Get()); - Stats.OutputChannels[channelId] = channel->GetStats(); - } - return channel; - } - + IDqOutputChannel::TPtr GetOutputChannel(ui64 channelId) override + { + auto& channel = OutputChannels[channelId]; + if (!channel) { + channel = new TDqOutputChannel( + Delegate->GetOutputChannel(channelId), + Task.GetId(), + channelId, + Delegate.Get()); + Stats.OutputChannels[channelId] = channel->GetStats(); + } + return channel; + } + IDqSink::TPtr GetSink(ui64 outputIndex) override { - auto& sink = Sinks[outputIndex]; - if (!sink) { - sink = new TDqSink( - Task.GetId(), - outputIndex, - Delegate.Get()); - // Stats.Sinks[outputIndex] = sink->GetStats(); - } - return sink; + auto& sink = Sinks[outputIndex]; + if (!sink) { + sink = new TDqSink( + Task.GetId(), + outputIndex, + Delegate.Get()); + // Stats.Sinks[outputIndex] = sink->GetStats(); + } + return sink; } const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const override { return Delegate->GetTypeEnv(); - } - + } + const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const override { return Delegate->GetHolderFactory(); } @@ -1591,8 +1591,8 @@ public: TGuard<NKikimr::NMiniKQL::TScopedAlloc> BindAllocator(TMaybe<ui64> memoryLimit) override { return Delegate->BindAllocator(memoryLimit); - } - + } + bool IsAllocatorAttached() override { return Delegate->IsAllocatorAttached(); } @@ -1600,26 +1600,26 @@ public: void UpdateStats() override { } - const TDqTaskRunnerStats* GetStats() const override - { - try { - NDqProto::TCommandHeader header; - header.SetVersion(3); - header.SetCommand(NDqProto::TCommandHeader::GET_STATS); - header.SetTaskId(Task.GetId()); - header.Save(&Delegate->GetOutput()); - - NDqProto::TGetStatsResponse response; - response.Load(&Delegate->GetInput()); - - FromProto(&Stats, response.GetStats()); - - return &Stats; - } catch (...) { - Delegate->RaiseException(); - } - } - + const TDqTaskRunnerStats* GetStats() const override + { + try { + NDqProto::TCommandHeader header; + header.SetVersion(3); + header.SetCommand(NDqProto::TCommandHeader::GET_STATS); + header.SetTaskId(Task.GetId()); + header.Save(&Delegate->GetOutput()); + + NDqProto::TGetStatsResponse response; + response.Load(&Delegate->GetInput()); + + FromProto(&Stats, response.GetStats()); + + return &Stats; + } catch (...) { + Delegate->RaiseException(); + } + } + TString Save() const override { ythrow yexception() << "unimplemented"; } @@ -1629,279 +1629,279 @@ public: ythrow yexception() << "unimplemented"; } -private: - TIntrusivePtr<TTaskRunner> Delegate; - NDqProto::TDqTask Task; - mutable TDqTaskRunnerStats Stats; - - THashMap<ui64, IDqInputChannel::TPtr> InputChannels; +private: + TIntrusivePtr<TTaskRunner> Delegate; + NDqProto::TDqTask Task; + mutable TDqTaskRunnerStats Stats; + + THashMap<ui64, IDqInputChannel::TPtr> InputChannels; THashMap<ui64, IDqSource::TPtr> Sources; - THashMap<ui64, IDqOutputChannel::TPtr> OutputChannels; - THashMap<ui64, IDqSink::TPtr> Sinks; -}; - -/*______________________________________________________________________________________________*/ - - -class TPipeFactory: public IProxyFactory { - struct TJob: public TTaskScheduler::ITask { - TJob(NThreading::TPromise<void> p) - : Promise(std::move(p)) - { } - - TInstant Process() override { - Promise.SetValue(); - return TInstant::Max(); - } - - NThreading::TPromise<void> Promise; - }; - - struct TStopJob: public TTaskScheduler::ITask { - TList<THolder<TChildProcess>> StopList; - - TStopJob(TList<THolder<TChildProcess>>&& stopList) - : StopList(std::move(stopList)) - { } - - TInstant Process() override { - for (const auto& job : StopList) { - job->Kill(); - job->Wait(TDuration::Seconds(1)); - } - - return TInstant::Max(); - } - }; - -public: - TPipeFactory(const TPipeFactoryOptions& options) - : ExePath(options.ExecPath) - , EnablePorto(options.EnablePorto) - , PortoSettings({ - EnablePorto, - TDqSettings::TDefault::PortoMemoryLimit, - options.PortoLayer, - options.ContainerName - }) - , FileCache(options.FileCache) - , Args {"yql@child", "tasks_runner_proxy"} - , Env(options.Env) - , Revision(GetProgramCommitId()) - , TaskScheduler(1) - , MaxProcesses(options.MaxProcesses) - , PortoCtlPath(options.PortoCtlPath) - { - Start(ExePath, PortoSettings); - TaskScheduler.Start(); - } - - ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& tmp, const TString& traceId) override { - Yql::DqsProto::TTaskMeta taskMeta; - tmp.GetMeta().UnpackTo(&taskMeta); - ui64 stageId = taskMeta.GetStageId(); - auto result = GetExecutorForTask(taskMeta.GetFiles(), taskMeta.GetSettings()); - auto task = PrepareTask(tmp, result.Get()); - return new TTaskRunner(task, std::move(result), stageId, traceId); - } - - TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& tmp, const TString& traceId) override - { - Yql::DqsProto::TTaskMeta taskMeta; - tmp.GetMeta().UnpackTo(&taskMeta); - - auto result = GetExecutorForTask(taskMeta.GetFiles(), taskMeta.GetSettings()); - auto task = PrepareTask(tmp, result.Get()); - return new TDqTaskRunner(task, std::move(result), taskMeta.GetStageId(), traceId); - } - -private: - THolder<TChildProcess> StartOne(const TString& exePath, const TPortoSettings& portoSettings) { - return CreateChildProcess(PortoCtlPath, FileCache->GetDir(), exePath, Args, Env, ContainerId++, portoSettings); - } - - void Start(const TString& exePath, const TPortoSettings& portoSettings) { - const auto key = GetKey(exePath, portoSettings); - while (static_cast<int>(ProcessHolder.Size()) < MaxProcesses) { - auto process = StartOne(exePath, portoSettings); - ProcessHolder.Put(key, std::move(process)); - } - } - - void ProcessJobs(const TString& exePath, const TPortoSettings& portoSettings) { - NThreading::TPromise<void> promise = NThreading::NewPromise(); - - promise.GetFuture().Apply([=](const NThreading::TFuture<void>&) mutable { - Start(exePath, portoSettings); - }); - - Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TJob>(promise), TInstant())); - } - - void StopJobs(TList<THolder<TChildProcess>>&& stopList) { - Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TStopJob>(std::move(stopList)), TInstant())); - } - - TString GetKey(const TString& exePath, const TPortoSettings& settings) - { - return exePath + "," + settings.ToString(); - } - - NDqProto::TDqTask PrepareTask(const NDqProto::TDqTask& tmp, TChildProcess* result) { - // get files from fileCache - NDqProto::TDqTask task = tmp; - Yql::DqsProto::TTaskMeta taskMeta; - - task.GetMeta().UnpackTo(&taskMeta); - - auto* files = taskMeta.MutableFiles(); - THashSet<TString> initialized; - - for (auto& file : *files) { - if (file.GetObjectType() != Yql::DqsProto::TFile::EEXE_FILE) { - if (initialized.contains(file.GetObjectId())) { - file.SetObjectType(Yql::DqsProto::TFile::EEXE_FILE); // skip this in runner - continue; - } - initialized.insert(file.GetObjectId()); - - auto maybeFile = FileCache->FindFile(file.GetObjectId()); - if (!maybeFile) { - throw std::runtime_error("Cannot find object `" + file.GetObjectId() + "' in cache"); - } - auto name = file.GetName(); - - switch (file.GetObjectType()) { - case Yql::DqsProto::TFile::EEXE_FILE: - break; - case Yql::DqsProto::TFile::EUDF_FILE: - case Yql::DqsProto::TFile::EUSER_FILE: - file.SetLocalPath(InitializeLocalFile(result->ExternalWorkDir(), *maybeFile, name)); - break; - default: - Y_VERIFY(false); - } - } - } - - auto& taskParams = *taskMeta.MutableTaskParams(); - taskParams[WorkingDirectoryParamName] = result->InternalWorkDir(); - taskParams[WorkingDirectoryDontInitParamName] = "true"; - task.MutableMeta()->PackFrom(taskMeta); - - return task; - } - - TFsPath MakeLocalPath(TString fileName) { - TString localPath = fileName; - if (localPath.StartsWith(TStringBuf("/home/"))) { - localPath = localPath.substr(TStringBuf("/home").length()); // Keep leading slash - } - if (!localPath.StartsWith('/')) { - localPath.prepend('/'); - } - localPath.prepend('.'); - return localPath; - } - - TString InitializeLocalFile(const TString& base, const TString& path, const TFsPath& name) - { - auto localPath = MakeLocalPath(name.GetPath()); - NFs::MakeDirectoryRecursive(base + "/" + localPath.Parent().GetPath(), NFs::FP_NONSECRET_FILE, false); - YQL_LOG(DEBUG) << "HardLink '" << path << "-> '" << (base + "/" + localPath.GetPath()) << "'"; - YQL_ENSURE(NFs::HardLink(path, base + "/" + localPath.GetPath())); - return localPath.GetPath(); - } - - template<typename T, typename S> - THolder<TChildProcess> GetExecutorForTask(const T& files, const S& settings) { - TString executorId; - TPortoSettings portoSettings = PortoSettings; - - for (const auto& file : files) { - if (file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE) { - executorId = file.GetObjectId(); - break; - } - } - - TDqConfiguration::TPtr conf = new TDqConfiguration; - try { - conf->Dispatch(settings); - } catch (...) { /* ignore unknown settings */ } - portoSettings.Enable = EnablePorto && conf->EnablePorto.Get().GetOrElse(TDqSettings::TDefault::EnablePorto); - portoSettings.MemoryLimit = conf->_PortoMemoryLimit.Get().GetOrElse(TDqSettings::TDefault::PortoMemoryLimit); - if (portoSettings.Enable) { - YQL_LOG(DEBUG) << "Porto enabled"; - } - - TString exePath; - if (executorId.empty() || executorId == Revision) { - exePath = ExePath; - } else { - auto maybeExeFile = FileCache->FindFile(executorId); - if (!maybeExeFile) { - throw std::runtime_error("Cannot find exe `" + executorId + "' in cache"); - } - exePath = *maybeExeFile; - } - - auto key = GetKey(exePath, portoSettings); - TList<THolder<TChildProcess>> stopList; - THolder<TChildProcess> result = ProcessHolder.Acquire(key, &stopList); - if (!result) { - result = StartOne(exePath, portoSettings); - } - ProcessJobs(exePath, portoSettings); - StopJobs(std::move(stopList)); - return result; - } - - static THolder<TChildProcess> CreateChildProcess( - const TString& portoCtlPath, - const TString& cacheDir, - const TString& exePath, - const TVector<TString>& args, - const THashMap<TString, TString>& env, - i64 containerId, - const TPortoSettings& portoSettings) - { - THolder<TChildProcess> command; - if (portoSettings.Enable) { - command = MakeHolder<TPortoProcess>(portoCtlPath, exePath, args, env, cacheDir + "/Slot-" + ToString(containerId), portoSettings); - } else { - command = MakeHolder<TChildProcess>(exePath, args, env, cacheDir + "/Slot-" + ToString(containerId)); - } - YQL_LOG(DEBUG) << "Executing " << exePath; - for (const auto& arg: args) { - YQL_LOG(DEBUG) << "Arg: " << arg; - } - command->Run(); + THashMap<ui64, IDqOutputChannel::TPtr> OutputChannels; + THashMap<ui64, IDqSink::TPtr> Sinks; +}; + +/*______________________________________________________________________________________________*/ + + +class TPipeFactory: public IProxyFactory { + struct TJob: public TTaskScheduler::ITask { + TJob(NThreading::TPromise<void> p) + : Promise(std::move(p)) + { } + + TInstant Process() override { + Promise.SetValue(); + return TInstant::Max(); + } + + NThreading::TPromise<void> Promise; + }; + + struct TStopJob: public TTaskScheduler::ITask { + TList<THolder<TChildProcess>> StopList; + + TStopJob(TList<THolder<TChildProcess>>&& stopList) + : StopList(std::move(stopList)) + { } + + TInstant Process() override { + for (const auto& job : StopList) { + job->Kill(); + job->Wait(TDuration::Seconds(1)); + } + + return TInstant::Max(); + } + }; + +public: + TPipeFactory(const TPipeFactoryOptions& options) + : ExePath(options.ExecPath) + , EnablePorto(options.EnablePorto) + , PortoSettings({ + EnablePorto, + TDqSettings::TDefault::PortoMemoryLimit, + options.PortoLayer, + options.ContainerName + }) + , FileCache(options.FileCache) + , Args {"yql@child", "tasks_runner_proxy"} + , Env(options.Env) + , Revision(GetProgramCommitId()) + , TaskScheduler(1) + , MaxProcesses(options.MaxProcesses) + , PortoCtlPath(options.PortoCtlPath) + { + Start(ExePath, PortoSettings); + TaskScheduler.Start(); + } + + ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& tmp, const TString& traceId) override { + Yql::DqsProto::TTaskMeta taskMeta; + tmp.GetMeta().UnpackTo(&taskMeta); + ui64 stageId = taskMeta.GetStageId(); + auto result = GetExecutorForTask(taskMeta.GetFiles(), taskMeta.GetSettings()); + auto task = PrepareTask(tmp, result.Get()); + return new TTaskRunner(task, std::move(result), stageId, traceId); + } + + TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& tmp, const TString& traceId) override + { + Yql::DqsProto::TTaskMeta taskMeta; + tmp.GetMeta().UnpackTo(&taskMeta); + + auto result = GetExecutorForTask(taskMeta.GetFiles(), taskMeta.GetSettings()); + auto task = PrepareTask(tmp, result.Get()); + return new TDqTaskRunner(task, std::move(result), taskMeta.GetStageId(), traceId); + } + +private: + THolder<TChildProcess> StartOne(const TString& exePath, const TPortoSettings& portoSettings) { + return CreateChildProcess(PortoCtlPath, FileCache->GetDir(), exePath, Args, Env, ContainerId++, portoSettings); + } + + void Start(const TString& exePath, const TPortoSettings& portoSettings) { + const auto key = GetKey(exePath, portoSettings); + while (static_cast<int>(ProcessHolder.Size()) < MaxProcesses) { + auto process = StartOne(exePath, portoSettings); + ProcessHolder.Put(key, std::move(process)); + } + } + + void ProcessJobs(const TString& exePath, const TPortoSettings& portoSettings) { + NThreading::TPromise<void> promise = NThreading::NewPromise(); + + promise.GetFuture().Apply([=](const NThreading::TFuture<void>&) mutable { + Start(exePath, portoSettings); + }); + + Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TJob>(promise), TInstant())); + } + + void StopJobs(TList<THolder<TChildProcess>>&& stopList) { + Y_VERIFY(TaskScheduler.Add(MakeIntrusive<TStopJob>(std::move(stopList)), TInstant())); + } + + TString GetKey(const TString& exePath, const TPortoSettings& settings) + { + return exePath + "," + settings.ToString(); + } + + NDqProto::TDqTask PrepareTask(const NDqProto::TDqTask& tmp, TChildProcess* result) { + // get files from fileCache + NDqProto::TDqTask task = tmp; + Yql::DqsProto::TTaskMeta taskMeta; + + task.GetMeta().UnpackTo(&taskMeta); + + auto* files = taskMeta.MutableFiles(); + THashSet<TString> initialized; + + for (auto& file : *files) { + if (file.GetObjectType() != Yql::DqsProto::TFile::EEXE_FILE) { + if (initialized.contains(file.GetObjectId())) { + file.SetObjectType(Yql::DqsProto::TFile::EEXE_FILE); // skip this in runner + continue; + } + initialized.insert(file.GetObjectId()); + + auto maybeFile = FileCache->FindFile(file.GetObjectId()); + if (!maybeFile) { + throw std::runtime_error("Cannot find object `" + file.GetObjectId() + "' in cache"); + } + auto name = file.GetName(); + + switch (file.GetObjectType()) { + case Yql::DqsProto::TFile::EEXE_FILE: + break; + case Yql::DqsProto::TFile::EUDF_FILE: + case Yql::DqsProto::TFile::EUSER_FILE: + file.SetLocalPath(InitializeLocalFile(result->ExternalWorkDir(), *maybeFile, name)); + break; + default: + Y_VERIFY(false); + } + } + } + + auto& taskParams = *taskMeta.MutableTaskParams(); + taskParams[WorkingDirectoryParamName] = result->InternalWorkDir(); + taskParams[WorkingDirectoryDontInitParamName] = "true"; + task.MutableMeta()->PackFrom(taskMeta); + + return task; + } + + TFsPath MakeLocalPath(TString fileName) { + TString localPath = fileName; + if (localPath.StartsWith(TStringBuf("/home/"))) { + localPath = localPath.substr(TStringBuf("/home").length()); // Keep leading slash + } + if (!localPath.StartsWith('/')) { + localPath.prepend('/'); + } + localPath.prepend('.'); + return localPath; + } + + TString InitializeLocalFile(const TString& base, const TString& path, const TFsPath& name) + { + auto localPath = MakeLocalPath(name.GetPath()); + NFs::MakeDirectoryRecursive(base + "/" + localPath.Parent().GetPath(), NFs::FP_NONSECRET_FILE, false); + YQL_LOG(DEBUG) << "HardLink '" << path << "-> '" << (base + "/" + localPath.GetPath()) << "'"; + YQL_ENSURE(NFs::HardLink(path, base + "/" + localPath.GetPath())); + return localPath.GetPath(); + } + + template<typename T, typename S> + THolder<TChildProcess> GetExecutorForTask(const T& files, const S& settings) { + TString executorId; + TPortoSettings portoSettings = PortoSettings; + + for (const auto& file : files) { + if (file.GetObjectType() == Yql::DqsProto::TFile::EEXE_FILE) { + executorId = file.GetObjectId(); + break; + } + } + + TDqConfiguration::TPtr conf = new TDqConfiguration; + try { + conf->Dispatch(settings); + } catch (...) { /* ignore unknown settings */ } + portoSettings.Enable = EnablePorto && conf->EnablePorto.Get().GetOrElse(TDqSettings::TDefault::EnablePorto); + portoSettings.MemoryLimit = conf->_PortoMemoryLimit.Get().GetOrElse(TDqSettings::TDefault::PortoMemoryLimit); + if (portoSettings.Enable) { + YQL_LOG(DEBUG) << "Porto enabled"; + } + + TString exePath; + if (executorId.empty() || executorId == Revision) { + exePath = ExePath; + } else { + auto maybeExeFile = FileCache->FindFile(executorId); + if (!maybeExeFile) { + throw std::runtime_error("Cannot find exe `" + executorId + "' in cache"); + } + exePath = *maybeExeFile; + } + + auto key = GetKey(exePath, portoSettings); + TList<THolder<TChildProcess>> stopList; + THolder<TChildProcess> result = ProcessHolder.Acquire(key, &stopList); + if (!result) { + result = StartOne(exePath, portoSettings); + } + ProcessJobs(exePath, portoSettings); + StopJobs(std::move(stopList)); + return result; + } + + static THolder<TChildProcess> CreateChildProcess( + const TString& portoCtlPath, + const TString& cacheDir, + const TString& exePath, + const TVector<TString>& args, + const THashMap<TString, TString>& env, + i64 containerId, + const TPortoSettings& portoSettings) + { + THolder<TChildProcess> command; + if (portoSettings.Enable) { + command = MakeHolder<TPortoProcess>(portoCtlPath, exePath, args, env, cacheDir + "/Slot-" + ToString(containerId), portoSettings); + } else { + command = MakeHolder<TChildProcess>(exePath, args, env, cacheDir + "/Slot-" + ToString(containerId)); + } + YQL_LOG(DEBUG) << "Executing " << exePath; + for (const auto& arg: args) { + YQL_LOG(DEBUG) << "Arg: " << arg; + } + command->Run(); return command; - } - - std::atomic<i64> ContainerId = 1; - - const TString ExePath; - const bool EnablePorto; - const TPortoSettings PortoSettings; - const IFileCache::TPtr FileCache; - const TString UdfsDir; - const TVector<TString> Args; - const THashMap<TString, TString> Env; - - const TString Revision; - - TProcessHolder ProcessHolder; - TTaskScheduler TaskScheduler; - const int MaxProcesses; - const TString PortoCtlPath; -}; - -IProxyFactory::TPtr CreatePipeFactory( - const TPipeFactoryOptions& options) -{ - return new TPipeFactory(options); -} - + } + + std::atomic<i64> ContainerId = 1; + + const TString ExePath; + const bool EnablePorto; + const TPortoSettings PortoSettings; + const IFileCache::TPtr FileCache; + const TString UdfsDir; + const TVector<TString> Args; + const THashMap<TString, TString> Env; + + const TString Revision; + + TProcessHolder ProcessHolder; + TTaskScheduler TaskScheduler; + const int MaxProcesses; + const TString PortoCtlPath; +}; + +IProxyFactory::TPtr CreatePipeFactory( + const TPipeFactoryOptions& options) +{ + return new TPipeFactory(options); +} + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h index b3b9eb5d66..137a0e8c0c 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h @@ -1,24 +1,24 @@ -#pragma once - +#pragma once + #include "tasks_runner_proxy.h" -#include "file_cache.h" - +#include "file_cache.h" + #include <util/generic/hash.h> #include <util/generic/string.h> - + namespace NYql::NTaskRunnerProxy { - -struct TPipeFactoryOptions { - TString ExecPath; - IFileCache::TPtr FileCache; - THashMap<TString, TString> Env; - bool EnablePorto = false; - TString PortoLayer; - int MaxProcesses = 1; - TString ContainerName; - TString PortoCtlPath = "/usr/bin/porto"; -}; - -IProxyFactory::TPtr CreatePipeFactory(const TPipeFactoryOptions& options); - + +struct TPipeFactoryOptions { + TString ExecPath; + IFileCache::TPtr FileCache; + THashMap<TString, TString> Env; + bool EnablePorto = false; + TString PortoLayer; + int MaxProcesses = 1; + TString ContainerName; + TString PortoCtlPath = "/usr/bin/porto"; +}; + +IProxyFactory::TPtr CreatePipeFactory(const TPipeFactoryOptions& options); + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp index 391fc49842..67174b88e9 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp @@ -1,15 +1,15 @@ -#include "tasks_runner_proxy.h" - +#include "tasks_runner_proxy.h" + namespace NYql::NTaskRunnerProxy { - -using namespace NKikimr; -using namespace NDq; - -TDqTaskRunnerMemoryLimits DefaultMemoryLimits() { - TDqTaskRunnerMemoryLimits limits; + +using namespace NKikimr; +using namespace NDq; + +TDqTaskRunnerMemoryLimits DefaultMemoryLimits() { + TDqTaskRunnerMemoryLimits limits; limits.ChannelBufferSize = 20_MB; limits.OutputChunkMaxSize = 2_MB; - return limits; -} - + return limits; +} + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h index e3f484355f..31a6e82784 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h @@ -1,71 +1,71 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/api/protos/task_command_executor.pb.h> #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node.h> - + namespace NYql::NTaskRunnerProxy { - -extern const TString WorkingDirectoryParamName; -extern const TString WorkingDirectoryDontInitParamName; // COMPAT(aozeritsky) -extern const TString UseMetaParamName; // COMPAT(aozeritsky) - -class IStringSource: public NDq::IDqSource { -public: - virtual ~IStringSource() = default; - virtual void PushString(TVector<TString>&& batch, i64 space) = 0; -}; - -class IStringSink: public NDq::IDqSink { -public: - virtual ~IStringSink() = default; - virtual ui64 PopString(TVector<TString>& batch, ui64 bytes) = 0; -}; - -class IInputChannel : public TThrRefBase, private TNonCopyable { -public: - using TPtr = TIntrusivePtr<IInputChannel>; - - virtual ~IInputChannel() = default; - - virtual void Push(NDqProto::TData&& data) = 0; - - virtual i64 GetFreeSpace() = 0; - - virtual void Finish() = 0; -}; - -class IOutputChannel : public TThrRefBase, private TNonCopyable { -public: - using TPtr = TIntrusivePtr<IOutputChannel>; - - virtual ~IOutputChannel() = default; - - [[nodiscard]] - virtual NDqProto::TPopResponse Pop(NDqProto::TData& data, ui64 bytes) = 0; - - virtual bool IsFinished() const = 0; -}; - -class ITaskRunner: public TThrRefBase, private TNonCopyable { -public: - using TPtr = TIntrusivePtr<ITaskRunner>; - - virtual ~ITaskRunner() = default; - + +extern const TString WorkingDirectoryParamName; +extern const TString WorkingDirectoryDontInitParamName; // COMPAT(aozeritsky) +extern const TString UseMetaParamName; // COMPAT(aozeritsky) + +class IStringSource: public NDq::IDqSource { +public: + virtual ~IStringSource() = default; + virtual void PushString(TVector<TString>&& batch, i64 space) = 0; +}; + +class IStringSink: public NDq::IDqSink { +public: + virtual ~IStringSink() = default; + virtual ui64 PopString(TVector<TString>& batch, ui64 bytes) = 0; +}; + +class IInputChannel : public TThrRefBase, private TNonCopyable { +public: + using TPtr = TIntrusivePtr<IInputChannel>; + + virtual ~IInputChannel() = default; + + virtual void Push(NDqProto::TData&& data) = 0; + + virtual i64 GetFreeSpace() = 0; + + virtual void Finish() = 0; +}; + +class IOutputChannel : public TThrRefBase, private TNonCopyable { +public: + using TPtr = TIntrusivePtr<IOutputChannel>; + + virtual ~IOutputChannel() = default; + + [[nodiscard]] + virtual NDqProto::TPopResponse Pop(NDqProto::TData& data, ui64 bytes) = 0; + + virtual bool IsFinished() const = 0; +}; + +class ITaskRunner: public TThrRefBase, private TNonCopyable { +public: + using TPtr = TIntrusivePtr<ITaskRunner>; + + virtual ~ITaskRunner() = default; + virtual ui64 GetTaskId() const = 0; - virtual NYql::NDqProto::TPrepareResponse Prepare() = 0; - virtual NYql::NDqProto::TRunResponse Run() = 0; - - virtual IInputChannel::TPtr GetInputChannel(ui64 channelId) = 0; - virtual IOutputChannel::TPtr GetOutputChannel(ui64 channelId) = 0; - virtual NDq::IDqSource::TPtr GetSource(ui64 index) = 0; - virtual NDq::IDqSink::TPtr GetSink(ui64 index) = 0; - - virtual const THashMap<TString,TString>& GetTaskParams() const = 0; - virtual const THashMap<TString,TString>& GetSecureParams() const = 0; + virtual NYql::NDqProto::TPrepareResponse Prepare() = 0; + virtual NYql::NDqProto::TRunResponse Run() = 0; + + virtual IInputChannel::TPtr GetInputChannel(ui64 channelId) = 0; + virtual IOutputChannel::TPtr GetOutputChannel(ui64 channelId) = 0; + virtual NDq::IDqSource::TPtr GetSource(ui64 index) = 0; + virtual NDq::IDqSink::TPtr GetSink(ui64 index) = 0; + + virtual const THashMap<TString,TString>& GetTaskParams() const = 0; + virtual const THashMap<TString,TString>& GetSecureParams() const = 0; virtual const NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnv() const = 0; virtual const NKikimr::NMiniKQL::THolderFactory& GetHolderFactory() const = 0; @@ -76,25 +76,25 @@ public: virtual bool IsAllocatorAttached() = 0; struct TStatus { - int ExitCode; - TString Stderr; - }; - - virtual i32 GetProtocolVersion() = 0; - virtual TStatus GetStatus() = 0; - virtual void Kill() { } -}; - -class IProxyFactory: public TThrRefBase, private TNonCopyable { -public: - using TPtr = TIntrusivePtr<IProxyFactory>; - - virtual ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& task, const TString& traceId = "") = 0; - - virtual TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& task, const TString& traceId = "TODO") = 0; -}; - - -NDq::TDqTaskRunnerMemoryLimits DefaultMemoryLimits(); - + int ExitCode; + TString Stderr; + }; + + virtual i32 GetProtocolVersion() = 0; + virtual TStatus GetStatus() = 0; + virtual void Kill() { } +}; + +class IProxyFactory: public TThrRefBase, private TNonCopyable { +public: + using TPtr = TIntrusivePtr<IProxyFactory>; + + virtual ITaskRunner::TPtr GetOld(const NDqProto::TDqTask& task, const TString& traceId = "") = 0; + + virtual TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& task, const TString& traceId = "TODO") = 0; +}; + + +NDq::TDqTaskRunnerMemoryLimits DefaultMemoryLimits(); + } // namespace NYql::NTaskRunnerProxy diff --git a/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.cpp b/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.cpp index 83c2f89160..9b4639718e 100644 --- a/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.cpp +++ b/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.cpp @@ -1,469 +1,469 @@ -#include "task_runner_actor.h" - +#include "task_runner_actor.h" + #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h> - + #include <ydb/library/yql/minikql/mkql_string_util.h> - -#include <library/cpp/actors/core/hfunc.h> - -using namespace NYql::NDqs; -using namespace NActors; - -namespace NYql::NDq { - -namespace NTaskRunnerActor { - -namespace { -template<typename T> -TTaskRunnerActorSensors GetSensors(const T& t) { - TTaskRunnerActorSensors result; - for (const auto& m : t.GetMetric()) { - result.push_back( - { - m.GetName(), m.GetSum(), m.GetMax(), m.GetMin(), m.GetAvg(), m.GetCount() - }); - } - return result; -} - -template<typename T> -TTaskRunnerActorRusage GetRusage(const T& t) { - TTaskRunnerActorRusage rusage = { - t.GetRusage().GetUtime(), - t.GetRusage().GetStime(), - t.GetRusage().GetMajorPageFaults() - }; - return rusage; -} - -} // namespace - -class TTaskRunnerActor - : public TActor<TTaskRunnerActor> - , public ITaskRunnerActor -{ -public: - static constexpr char ActorName[] = "YQL_DQ_TASK_RUNNER"; - - TTaskRunnerActor( - ITaskRunnerActor::ICallbacks* parent, - const NTaskRunnerProxy::IProxyFactory::TPtr& factory, - const ITaskRunnerInvoker::TPtr& invoker, - const TString& traceId) - : TActor<TTaskRunnerActor>(&TTaskRunnerActor::Handler) - , Parent(parent) - , TraceId(traceId) - , Factory(factory) - , Invoker(invoker) - , Local(Invoker->IsLocal()) - { } - - ~TTaskRunnerActor() { } - - STRICT_STFUNC(Handler, { - cFunc(NActors::TEvents::TEvPoison::EventType, TTaskRunnerActor::PassAway); - HFunc(TEvTaskRunnerCreate, OnDqTask); - HFunc(TEvContinueRun, OnContinueRun); - HFunc(TEvPop, OnChannelPop); - HFunc(TEvPush, OnChannelPush); - HFunc(TEvSinkPop, OnSinkPop); - HFunc(TEvSinkPopFinished, OnSinkPopFinished); - }) - -private: - void PassAway() override { - if (TaskRunner) { - Invoker->Invoke([taskRunner=std::move(TaskRunner)] () { - taskRunner->Kill(); - }); - TaskRunner.Reset(); - } - TActor<TTaskRunnerActor>::PassAway(); - } - - void OnContinueRun(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { - Run(ev, ctx); - } - - void OnChannelPush(TEvPush::TPtr& ev, const NActors::TActorContext& ctx) { - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - auto replyTo = ev->Sender; - auto selfId = SelfId(); - auto hasData = ev->Get()->HasData; - auto finish = ev->Get()->Finish; - auto askFreeSpace = ev->Get()->AskFreeSpace; - auto channelId = ev->Get()->ChannelId; - auto cookie = ev->Cookie; - auto data = ev->Get()->Data; - Invoker->Invoke([hasData, selfId, cookie, askFreeSpace, finish, channelId, taskRunner=TaskRunner, data, actorSystem, replyTo] () mutable { - try { - ui64 freeSpace = 0; - if (hasData) { - // auto guard = taskRunner->BindAllocator(); // only for local mode - taskRunner->GetInputChannel(channelId)->Push(std::move(data)); - if (askFreeSpace) { - freeSpace = taskRunner->GetInputChannel(channelId)->GetFreeSpace(); - } - } - if (finish) { - taskRunner->GetInputChannel(channelId)->Finish(); - } - - // run - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvContinueRun(channelId, freeSpace), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), - /*flags=*/0, - cookie)); - } - }); - } - - void SourcePush( - ui64 cookie, - ui64 index, - NKikimr::NMiniKQL::TUnboxedValueVector&& batch, - i64 space, - bool finish) override - { - auto* actorSystem = NActors::TlsActivationContext->ExecutorThread.ActorSystem; - auto selfId = SelfId(); - - TVector<TString> strings; - for (auto& row : batch) { - strings.emplace_back(row.AsStringRef()); - } - - Invoker->Invoke([strings=std::move(strings),taskRunner=TaskRunner, actorSystem, selfId, cookie, parentId=ParentId, space, finish, index]() mutable { - try { - // auto guard = taskRunner->BindAllocator(); // only for local mode - auto source = taskRunner->GetSource(index); - (static_cast<NTaskRunnerProxy::IStringSource*>(source.Get()))->PushString(std::move(strings), space); - if (finish) { - source->Finish(); - } - actorSystem->Send( - new IEventHandle( - parentId, - selfId, - new TEvSourcePushFinished(index), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle( - parentId, - selfId, - new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), - /*flags=*/0, - cookie)); - } - }); - } - - void OnChannelPop(TEvPop::TPtr& ev, const NActors::TActorContext& ctx) { - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - auto replyTo = ev->Sender; - auto selfId = SelfId(); - auto cookie = ev->Cookie; - auto wasFinished = ev->Get()->WasFinished; - auto toPop = ev->Get()->Size; - Invoker->Invoke([cookie,selfId,channelId=ev->Get()->ChannelId, actorSystem, replyTo, wasFinished, toPop, taskRunner=TaskRunner]() { - try { - // auto guard = taskRunner->BindAllocator(); // only for local mode - auto channel = taskRunner->GetOutputChannel(channelId); - int maxChunks = std::numeric_limits<int>::max(); - bool changed = false; - bool isFinished = false; - i64 remain = toPop; - ui32 dataSize = 0; - bool hasData = true; - - if (remain == 0) { - // special case to WorkerActor - remain = 5<<20; - maxChunks = 1; - } - - TVector<NDqProto::TData> chunks; - NDqProto::TPopResponse response; - for (;maxChunks && remain > 0 && !isFinished && hasData; maxChunks--, remain -= dataSize) { - NDqProto::TData data; - NDqProto::TPopResponse pop = channel->Pop(data, remain); - - for (auto& metric : pop.GetMetric()) { - *response.AddMetric() = metric; - } - - hasData = pop.GetResult(); - dataSize = data.GetRaw().size(); - isFinished = !hasData && channel->IsFinished(); - response.SetResult(response.GetResult() || hasData); - changed = changed || hasData || (isFinished != wasFinished); - - if (hasData) { - chunks.emplace_back(std::move(data)); - } - } - - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvChannelPopFinished( - channelId, - std::move(chunks), - isFinished, - changed, - GetSensors(response)), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), - /*flags=*/0, - cookie)); - } - }); - } - - void OnSinkPopFinished(TEvSinkPopFinished::TPtr& ev, const NActors::TActorContext& ctx) { - Y_UNUSED(ctx); - auto guard = TaskRunner->BindAllocator(); - NKikimr::NMiniKQL::TUnboxedValueVector batch; - for (auto& row: ev->Get()->Strings) { - batch.emplace_back(NKikimr::NMiniKQL::MakeString(row)); - } - Parent->SinkSend( - ev->Get()->Index, - std::move(batch), - std::move(ev->Get()->Checkpoint), - ev->Get()->CheckpointSize, - ev->Get()->Size, - ev->Get()->Finished, - ev->Get()->Changed); - } - - void OnSinkPop(TEvSinkPop::TPtr& ev, const NActors::TActorContext& ctx) { - auto selfId = SelfId(); - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - - Invoker->Invoke([taskRunner=TaskRunner, selfId, actorSystem, ev=std::move(ev)] { - auto cookie = ev->Cookie; - auto replyTo = ev->Sender; - - try { - // auto guard = taskRunner->BindAllocator(); // only for local mode - auto sink = taskRunner->GetSink(ev->Get()->Index); - TVector<TString> batch; - NDqProto::TCheckpoint checkpoint; - TMaybe<NDqProto::TCheckpoint> maybeCheckpoint; - i64 size = 0; - i64 checkpointSize = 0; - if (ev->Get()->Size > 0) { - size = (static_cast<NTaskRunnerProxy::IStringSink*>(sink.Get()))->PopString(batch, ev->Get()->Size); - } - bool hasCheckpoint = sink->Pop(checkpoint); - if (hasCheckpoint) { - checkpointSize = checkpoint.ByteSize(); - maybeCheckpoint.ConstructInPlace(std::move(checkpoint)); - } - auto finished = sink->IsFinished(); - bool changed = finished || ev->Get()->Size > 0 || hasCheckpoint; - auto event = MakeHolder<TEvSinkPopFinished>( - ev->Get()->Index, - std::move(maybeCheckpoint), size, checkpointSize, finished, changed); - event->Strings = std::move(batch); - // repack data and forward - actorSystem->Send( - new IEventHandle( - selfId, - replyTo, - event.Release(), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), - /*flags=*/0, - cookie)); - } - }); - } - - void OnDqTask(TEvTaskRunnerCreate::TPtr& ev, const NActors::TActorContext& ctx) { - auto replyTo = ev->Sender; - auto selfId = SelfId(); - auto cookie = ev->Cookie; - auto taskId = ev->Get()->Task.GetId(); - auto& inputs = ev->Get()->Task.GetInputs(); - for (auto inputId = 0; inputId < inputs.size(); inputId++) { - auto& input = inputs[inputId]; - if (input.HasSource()) { - Sources.emplace(inputId); - } else { - for (auto& channel : input.GetChannels()) { - Inputs.emplace(channel.GetId()); - } - } - } - ParentId = ev->Sender; - - try { - TaskRunner = Factory->GetOld(ev->Get()->Task, TraceId); - } catch (...) { - TString message = "Could not create TaskRunner for " + ToString(taskId) + " on node " + ToString(replyTo.NodeId()) + ", error: " + CurrentExceptionMessage(); - Send(replyTo, new TEvError(message, /*retriable = */ true, /*fallback=*/ true), 0, cookie); - return; - } - - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - Invoker->Invoke([taskRunner=TaskRunner, replyTo, selfId, cookie, actorSystem](){ - try { - //auto guard = taskRunner->BindAllocator(); // only for local mode - auto result = taskRunner->Prepare(); - - auto event = MakeHolder<TEvTaskRunnerCreateFinished>( - taskRunner->GetSecureParams(), - taskRunner->GetTaskParams(), - taskRunner->GetTypeEnv(), - taskRunner->GetHolderFactory(), - GetSensors(result)); - - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - event.Release(), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle(replyTo, selfId, new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), 0, cookie)); - } - }); - } - - void Run(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - auto replyTo = ev->Sender; - auto selfId = SelfId(); - auto cookie = ev->Cookie; - auto inputMap = ev->Get()->AskFreeSpace - ? Inputs - : ev->Get()->InputChannels; - - auto sourcesMap = Sources; - - Invoker->Invoke([selfId, cookie, actorSystem, replyTo, taskRunner=TaskRunner, inputMap, sourcesMap, memLimit=ev->Get()->MemLimit]() mutable { - try { - // auto guard = taskRunner->BindAllocator(); // only for local mode - // guard.GetMutex()->SetLimit(memLimit); - auto response = taskRunner->Run(); - auto res = static_cast<NDq::ERunStatus>(response.GetResult()); - - THashMap<ui32, ui64> inputChannelFreeSpace; - THashMap<ui32, ui64> sourcesFreeSpace; - if (res == ERunStatus::PendingInput) { - for (auto& channelId : inputMap) { - inputChannelFreeSpace[channelId] = taskRunner->GetInputChannel(channelId)->GetFreeSpace(); - } - - for (auto& index : sourcesMap) { - sourcesFreeSpace[index] = taskRunner->GetSource(index)->GetFreeSpace(); - } - } - - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvTaskRunFinished( - res, - std::move(inputChannelFreeSpace), - std::move(sourcesFreeSpace), - GetSensors(response), - GetRusage(response)), - /*flags=*/0, - cookie)); - } catch (...) { - auto status = taskRunner->GetStatus(); - actorSystem->Send( - new IEventHandle( - replyTo, - selfId, - new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), - /*flags=*/0, - cookie)); - } - }); - } - - NActors::TActorId ParentId; - ITaskRunnerActor::ICallbacks* Parent; - TString TraceId; - NTaskRunnerProxy::IProxyFactory::TPtr Factory; - NTaskRunnerProxy::ITaskRunner::TPtr TaskRunner; - ITaskRunnerInvoker::TPtr Invoker; - bool Local; - THashSet<ui32> Inputs; - THashSet<ui32> Sources; -}; - -class TTaskRunnerActorFactory: public ITaskRunnerActorFactory { -public: - TTaskRunnerActorFactory( - const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, - const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory) - : ProxyFactory(proxyFactory) - , InvokerFactory(invokerFactory) - { } - - std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( - ITaskRunnerActor::ICallbacks* parent, - const TString& traceId) override - { - auto* actor = new TTaskRunnerActor(parent, ProxyFactory, InvokerFactory->Create(), traceId); - return std::make_tuple( - static_cast<ITaskRunnerActor*>(actor), - static_cast<NActors::IActor*>(actor) - ); - } - -private: - NTaskRunnerProxy::IProxyFactory::TPtr ProxyFactory; - NDqs::ITaskRunnerInvokerFactory::TPtr InvokerFactory; -}; - -ITaskRunnerActorFactory::TPtr CreateTaskRunnerActorFactory( - const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, - const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory) -{ - return ITaskRunnerActorFactory::TPtr(new TTaskRunnerActorFactory(proxyFactory, invokerFactory)); -} - -} // namespace NTaskRunnerActor - -} // namespace NYql::NDq + +#include <library/cpp/actors/core/hfunc.h> + +using namespace NYql::NDqs; +using namespace NActors; + +namespace NYql::NDq { + +namespace NTaskRunnerActor { + +namespace { +template<typename T> +TTaskRunnerActorSensors GetSensors(const T& t) { + TTaskRunnerActorSensors result; + for (const auto& m : t.GetMetric()) { + result.push_back( + { + m.GetName(), m.GetSum(), m.GetMax(), m.GetMin(), m.GetAvg(), m.GetCount() + }); + } + return result; +} + +template<typename T> +TTaskRunnerActorRusage GetRusage(const T& t) { + TTaskRunnerActorRusage rusage = { + t.GetRusage().GetUtime(), + t.GetRusage().GetStime(), + t.GetRusage().GetMajorPageFaults() + }; + return rusage; +} + +} // namespace + +class TTaskRunnerActor + : public TActor<TTaskRunnerActor> + , public ITaskRunnerActor +{ +public: + static constexpr char ActorName[] = "YQL_DQ_TASK_RUNNER"; + + TTaskRunnerActor( + ITaskRunnerActor::ICallbacks* parent, + const NTaskRunnerProxy::IProxyFactory::TPtr& factory, + const ITaskRunnerInvoker::TPtr& invoker, + const TString& traceId) + : TActor<TTaskRunnerActor>(&TTaskRunnerActor::Handler) + , Parent(parent) + , TraceId(traceId) + , Factory(factory) + , Invoker(invoker) + , Local(Invoker->IsLocal()) + { } + + ~TTaskRunnerActor() { } + + STRICT_STFUNC(Handler, { + cFunc(NActors::TEvents::TEvPoison::EventType, TTaskRunnerActor::PassAway); + HFunc(TEvTaskRunnerCreate, OnDqTask); + HFunc(TEvContinueRun, OnContinueRun); + HFunc(TEvPop, OnChannelPop); + HFunc(TEvPush, OnChannelPush); + HFunc(TEvSinkPop, OnSinkPop); + HFunc(TEvSinkPopFinished, OnSinkPopFinished); + }) + +private: + void PassAway() override { + if (TaskRunner) { + Invoker->Invoke([taskRunner=std::move(TaskRunner)] () { + taskRunner->Kill(); + }); + TaskRunner.Reset(); + } + TActor<TTaskRunnerActor>::PassAway(); + } + + void OnContinueRun(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { + Run(ev, ctx); + } + + void OnChannelPush(TEvPush::TPtr& ev, const NActors::TActorContext& ctx) { + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto replyTo = ev->Sender; + auto selfId = SelfId(); + auto hasData = ev->Get()->HasData; + auto finish = ev->Get()->Finish; + auto askFreeSpace = ev->Get()->AskFreeSpace; + auto channelId = ev->Get()->ChannelId; + auto cookie = ev->Cookie; + auto data = ev->Get()->Data; + Invoker->Invoke([hasData, selfId, cookie, askFreeSpace, finish, channelId, taskRunner=TaskRunner, data, actorSystem, replyTo] () mutable { + try { + ui64 freeSpace = 0; + if (hasData) { + // auto guard = taskRunner->BindAllocator(); // only for local mode + taskRunner->GetInputChannel(channelId)->Push(std::move(data)); + if (askFreeSpace) { + freeSpace = taskRunner->GetInputChannel(channelId)->GetFreeSpace(); + } + } + if (finish) { + taskRunner->GetInputChannel(channelId)->Finish(); + } + + // run + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvContinueRun(channelId, freeSpace), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), + /*flags=*/0, + cookie)); + } + }); + } + + void SourcePush( + ui64 cookie, + ui64 index, + NKikimr::NMiniKQL::TUnboxedValueVector&& batch, + i64 space, + bool finish) override + { + auto* actorSystem = NActors::TlsActivationContext->ExecutorThread.ActorSystem; + auto selfId = SelfId(); + + TVector<TString> strings; + for (auto& row : batch) { + strings.emplace_back(row.AsStringRef()); + } + + Invoker->Invoke([strings=std::move(strings),taskRunner=TaskRunner, actorSystem, selfId, cookie, parentId=ParentId, space, finish, index]() mutable { + try { + // auto guard = taskRunner->BindAllocator(); // only for local mode + auto source = taskRunner->GetSource(index); + (static_cast<NTaskRunnerProxy::IStringSource*>(source.Get()))->PushString(std::move(strings), space); + if (finish) { + source->Finish(); + } + actorSystem->Send( + new IEventHandle( + parentId, + selfId, + new TEvSourcePushFinished(index), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle( + parentId, + selfId, + new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), + /*flags=*/0, + cookie)); + } + }); + } + + void OnChannelPop(TEvPop::TPtr& ev, const NActors::TActorContext& ctx) { + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto replyTo = ev->Sender; + auto selfId = SelfId(); + auto cookie = ev->Cookie; + auto wasFinished = ev->Get()->WasFinished; + auto toPop = ev->Get()->Size; + Invoker->Invoke([cookie,selfId,channelId=ev->Get()->ChannelId, actorSystem, replyTo, wasFinished, toPop, taskRunner=TaskRunner]() { + try { + // auto guard = taskRunner->BindAllocator(); // only for local mode + auto channel = taskRunner->GetOutputChannel(channelId); + int maxChunks = std::numeric_limits<int>::max(); + bool changed = false; + bool isFinished = false; + i64 remain = toPop; + ui32 dataSize = 0; + bool hasData = true; + + if (remain == 0) { + // special case to WorkerActor + remain = 5<<20; + maxChunks = 1; + } + + TVector<NDqProto::TData> chunks; + NDqProto::TPopResponse response; + for (;maxChunks && remain > 0 && !isFinished && hasData; maxChunks--, remain -= dataSize) { + NDqProto::TData data; + NDqProto::TPopResponse pop = channel->Pop(data, remain); + + for (auto& metric : pop.GetMetric()) { + *response.AddMetric() = metric; + } + + hasData = pop.GetResult(); + dataSize = data.GetRaw().size(); + isFinished = !hasData && channel->IsFinished(); + response.SetResult(response.GetResult() || hasData); + changed = changed || hasData || (isFinished != wasFinished); + + if (hasData) { + chunks.emplace_back(std::move(data)); + } + } + + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvChannelPopFinished( + channelId, + std::move(chunks), + isFinished, + changed, + GetSensors(response)), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), + /*flags=*/0, + cookie)); + } + }); + } + + void OnSinkPopFinished(TEvSinkPopFinished::TPtr& ev, const NActors::TActorContext& ctx) { + Y_UNUSED(ctx); + auto guard = TaskRunner->BindAllocator(); + NKikimr::NMiniKQL::TUnboxedValueVector batch; + for (auto& row: ev->Get()->Strings) { + batch.emplace_back(NKikimr::NMiniKQL::MakeString(row)); + } + Parent->SinkSend( + ev->Get()->Index, + std::move(batch), + std::move(ev->Get()->Checkpoint), + ev->Get()->CheckpointSize, + ev->Get()->Size, + ev->Get()->Finished, + ev->Get()->Changed); + } + + void OnSinkPop(TEvSinkPop::TPtr& ev, const NActors::TActorContext& ctx) { + auto selfId = SelfId(); + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + + Invoker->Invoke([taskRunner=TaskRunner, selfId, actorSystem, ev=std::move(ev)] { + auto cookie = ev->Cookie; + auto replyTo = ev->Sender; + + try { + // auto guard = taskRunner->BindAllocator(); // only for local mode + auto sink = taskRunner->GetSink(ev->Get()->Index); + TVector<TString> batch; + NDqProto::TCheckpoint checkpoint; + TMaybe<NDqProto::TCheckpoint> maybeCheckpoint; + i64 size = 0; + i64 checkpointSize = 0; + if (ev->Get()->Size > 0) { + size = (static_cast<NTaskRunnerProxy::IStringSink*>(sink.Get()))->PopString(batch, ev->Get()->Size); + } + bool hasCheckpoint = sink->Pop(checkpoint); + if (hasCheckpoint) { + checkpointSize = checkpoint.ByteSize(); + maybeCheckpoint.ConstructInPlace(std::move(checkpoint)); + } + auto finished = sink->IsFinished(); + bool changed = finished || ev->Get()->Size > 0 || hasCheckpoint; + auto event = MakeHolder<TEvSinkPopFinished>( + ev->Get()->Index, + std::move(maybeCheckpoint), size, checkpointSize, finished, changed); + event->Strings = std::move(batch); + // repack data and forward + actorSystem->Send( + new IEventHandle( + selfId, + replyTo, + event.Release(), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), + /*flags=*/0, + cookie)); + } + }); + } + + void OnDqTask(TEvTaskRunnerCreate::TPtr& ev, const NActors::TActorContext& ctx) { + auto replyTo = ev->Sender; + auto selfId = SelfId(); + auto cookie = ev->Cookie; + auto taskId = ev->Get()->Task.GetId(); + auto& inputs = ev->Get()->Task.GetInputs(); + for (auto inputId = 0; inputId < inputs.size(); inputId++) { + auto& input = inputs[inputId]; + if (input.HasSource()) { + Sources.emplace(inputId); + } else { + for (auto& channel : input.GetChannels()) { + Inputs.emplace(channel.GetId()); + } + } + } + ParentId = ev->Sender; + + try { + TaskRunner = Factory->GetOld(ev->Get()->Task, TraceId); + } catch (...) { + TString message = "Could not create TaskRunner for " + ToString(taskId) + " on node " + ToString(replyTo.NodeId()) + ", error: " + CurrentExceptionMessage(); + Send(replyTo, new TEvError(message, /*retriable = */ true, /*fallback=*/ true), 0, cookie); + return; + } + + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + Invoker->Invoke([taskRunner=TaskRunner, replyTo, selfId, cookie, actorSystem](){ + try { + //auto guard = taskRunner->BindAllocator(); // only for local mode + auto result = taskRunner->Prepare(); + + auto event = MakeHolder<TEvTaskRunnerCreateFinished>( + taskRunner->GetSecureParams(), + taskRunner->GetTaskParams(), + taskRunner->GetTypeEnv(), + taskRunner->GetHolderFactory(), + GetSensors(result)); + + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + event.Release(), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle(replyTo, selfId, new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), 0, cookie)); + } + }); + } + + void Run(TEvContinueRun::TPtr& ev, const TActorContext& ctx) { + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + auto replyTo = ev->Sender; + auto selfId = SelfId(); + auto cookie = ev->Cookie; + auto inputMap = ev->Get()->AskFreeSpace + ? Inputs + : ev->Get()->InputChannels; + + auto sourcesMap = Sources; + + Invoker->Invoke([selfId, cookie, actorSystem, replyTo, taskRunner=TaskRunner, inputMap, sourcesMap, memLimit=ev->Get()->MemLimit]() mutable { + try { + // auto guard = taskRunner->BindAllocator(); // only for local mode + // guard.GetMutex()->SetLimit(memLimit); + auto response = taskRunner->Run(); + auto res = static_cast<NDq::ERunStatus>(response.GetResult()); + + THashMap<ui32, ui64> inputChannelFreeSpace; + THashMap<ui32, ui64> sourcesFreeSpace; + if (res == ERunStatus::PendingInput) { + for (auto& channelId : inputMap) { + inputChannelFreeSpace[channelId] = taskRunner->GetInputChannel(channelId)->GetFreeSpace(); + } + + for (auto& index : sourcesMap) { + sourcesFreeSpace[index] = taskRunner->GetSource(index)->GetFreeSpace(); + } + } + + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvTaskRunFinished( + res, + std::move(inputChannelFreeSpace), + std::move(sourcesFreeSpace), + GetSensors(response), + GetRusage(response)), + /*flags=*/0, + cookie)); + } catch (...) { + auto status = taskRunner->GetStatus(); + actorSystem->Send( + new IEventHandle( + replyTo, + selfId, + new TEvError(CurrentExceptionMessage(), {status.ExitCode, status.Stderr}), + /*flags=*/0, + cookie)); + } + }); + } + + NActors::TActorId ParentId; + ITaskRunnerActor::ICallbacks* Parent; + TString TraceId; + NTaskRunnerProxy::IProxyFactory::TPtr Factory; + NTaskRunnerProxy::ITaskRunner::TPtr TaskRunner; + ITaskRunnerInvoker::TPtr Invoker; + bool Local; + THashSet<ui32> Inputs; + THashSet<ui32> Sources; +}; + +class TTaskRunnerActorFactory: public ITaskRunnerActorFactory { +public: + TTaskRunnerActorFactory( + const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, + const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory) + : ProxyFactory(proxyFactory) + , InvokerFactory(invokerFactory) + { } + + std::tuple<ITaskRunnerActor*, NActors::IActor*> Create( + ITaskRunnerActor::ICallbacks* parent, + const TString& traceId) override + { + auto* actor = new TTaskRunnerActor(parent, ProxyFactory, InvokerFactory->Create(), traceId); + return std::make_tuple( + static_cast<ITaskRunnerActor*>(actor), + static_cast<NActors::IActor*>(actor) + ); + } + +private: + NTaskRunnerProxy::IProxyFactory::TPtr ProxyFactory; + NDqs::ITaskRunnerInvokerFactory::TPtr InvokerFactory; +}; + +ITaskRunnerActorFactory::TPtr CreateTaskRunnerActorFactory( + const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, + const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory) +{ + return ITaskRunnerActorFactory::TPtr(new TTaskRunnerActorFactory(proxyFactory, invokerFactory)); +} + +} // namespace NTaskRunnerActor + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h b/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h index 07dfbee8a7..103dbeabd6 100644 --- a/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h +++ b/ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h @@ -1,31 +1,31 @@ -#pragma once - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/event_local.h> -#include <library/cpp/actors/core/event_pb.h> - +#pragma once + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/event_local.h> +#include <library/cpp/actors/core/event_pb.h> + #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> #include <ydb/library/yql/minikql/mkql_node.h> - + #include <ydb/library/yql/dq/actors/task_runner/events.h> #include <ydb/library/yql/dq/actors/task_runner/task_runner_actor.h> #include <ydb/library/yql/dq/runtime/dq_tasks_runner.h> #include <ydb/library/yql/dq/common/dq_common.h> #include <ydb/library/yql/dq/proto/dq_transport.pb.h> #include <ydb/library/yql/dq/proto/dq_tasks.pb.h> - + #include <ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h> #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h> - -namespace NYql::NDq { - -namespace NTaskRunnerActor { - -ITaskRunnerActorFactory::TPtr CreateTaskRunnerActorFactory( - const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, - const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory); - -} // namespace NTaskRunnerActor - -} // namespace NYql::NDq + +namespace NYql::NDq { + +namespace NTaskRunnerActor { + +ITaskRunnerActorFactory::TPtr CreateTaskRunnerActorFactory( + const NTaskRunnerProxy::IProxyFactory::TPtr& proxyFactory, + const NDqs::ITaskRunnerInvokerFactory::TPtr& invokerFactory); + +} // namespace NTaskRunnerActor + +} // namespace NYql::NDq diff --git a/ydb/library/yql/providers/dq/task_runner_actor/ya.make b/ydb/library/yql/providers/dq/task_runner_actor/ya.make index 2cb42c01a5..b15885f2fb 100644 --- a/ydb/library/yql/providers/dq/task_runner_actor/ya.make +++ b/ydb/library/yql/providers/dq/task_runner_actor/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - -OWNER(g:yql) - -SRCS( - task_runner_actor.cpp -) - -PEERDIR( - library/cpp/actors/core +LIBRARY() + +OWNER(g:yql) + +SRCS( + task_runner_actor.cpp +) + +PEERDIR( + library/cpp/actors/core ydb/library/yql/dq/actors/task_runner ydb/library/yql/providers/dq/api/protos ydb/library/yql/utils/actors ydb/library/yql/providers/dq/task_runner -) - -YQL_LAST_ABI_VERSION() - -END() +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/counters.cpp b/ydb/library/yql/providers/dq/worker_manager/interface/counters.cpp index b86369db86..6b298a5641 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/counters.cpp +++ b/ydb/library/yql/providers/dq/worker_manager/interface/counters.cpp @@ -8,8 +8,8 @@ TWorkerManagerCounters::TWorkerManagerCounters(NMonitoring::TDynamicCounterPtr r MkqlMemoryAllocated = root->GetCounter("MkqlMemoryAllocated"); } -TWorkerManagerCounters::TWorkerManagerCounters() - : TWorkerManagerCounters(new NMonitoring::TDynamicCounters) -{ } - -} // namespace NYql::NDqs +TWorkerManagerCounters::TWorkerManagerCounters() + : TWorkerManagerCounters(new NMonitoring::TDynamicCounters) +{ } + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/counters.h b/ydb/library/yql/providers/dq/worker_manager/interface/counters.h index dcb3df51a4..51290cba62 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/counters.h +++ b/ydb/library/yql/providers/dq/worker_manager/interface/counters.h @@ -9,7 +9,7 @@ struct TWorkerManagerCounters { NMonitoring::TDynamicCounters::TCounterPtr MkqlMemoryAllocated; explicit TWorkerManagerCounters(NMonitoring::TDynamicCounterPtr root); - TWorkerManagerCounters(); + TWorkerManagerCounters(); }; -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/events.cpp b/ydb/library/yql/providers/dq/worker_manager/interface/events.cpp index 86fcff7bc9..9044dc47e3 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/events.cpp +++ b/ydb/library/yql/providers/dq/worker_manager/interface/events.cpp @@ -1,36 +1,36 @@ #include "events.h" -namespace NYql::NCommonAttrs { - extern TString OPERATIONID_ATTR; - extern TString JOBID_ATTR; -} - +namespace NYql::NCommonAttrs { + extern TString OPERATIONID_ATTR; + extern TString JOBID_ATTR; +} + namespace NYql::NDqs { - TEvAllocateWorkersRequest::TEvAllocateWorkersRequest( - ui32 count, + TEvAllocateWorkersRequest::TEvAllocateWorkersRequest( + ui32 count, const TString& user, - const TMaybe<ui64>& globalResourceId) - { + const TMaybe<ui64>& globalResourceId) + { Record.SetCount(count); Record.SetUser(user); - if (globalResourceId) { - Record.SetResourceId(*globalResourceId); - } - Record.SetIsForwarded(false); + if (globalResourceId) { + Record.SetResourceId(*globalResourceId); + } + Record.SetIsForwarded(false); } TEvAllocateWorkersResponse::TEvAllocateWorkersResponse() { - Record.MutableError()->SetMessage("Unknown error"); + Record.MutableError()->SetMessage("Unknown error"); + } + + TEvAllocateWorkersResponse::TEvAllocateWorkersResponse(const TString& error, NYql::NDqProto::EErrorCode code) { + Record.MutableError()->SetMessage(error); + Record.MutableError()->SetErrorCode(code); } - TEvAllocateWorkersResponse::TEvAllocateWorkersResponse(const TString& error, NYql::NDqProto::EErrorCode code) { - Record.MutableError()->SetMessage(error); - Record.MutableError()->SetErrorCode(code); - } - TEvAllocateWorkersResponse::TEvAllocateWorkersResponse(ui64 resourceId, const TVector<NActors::TActorId>& ids) { auto& group = *Record.MutableWorkers(); - group.SetResourceId(resourceId); + group.SetResourceId(resourceId); for (const auto& actorId : ids) { NActors::ActorIdToProto(actorId, group.AddWorkerActor()); } @@ -45,61 +45,61 @@ namespace NYql::NDqs { } } - TEvAllocateWorkersResponse::TEvAllocateWorkersResponse(ui64 resourceId, const TVector<TWorkerInfo::TPtr>& infos) { - auto& group = *Record.MutableNodes(); - group.SetResourceId(resourceId); - for (const auto& workerInfo : infos) { - auto* worker = group.AddWorker(); - *worker->MutableGuid() = GetGuidAsString(workerInfo->WorkerId); // for debug only - worker->SetNodeId(workerInfo->NodeId); - worker->SetClusterName(workerInfo->ClusterName); - - for (const auto& [k, v] : workerInfo->Attributes) { - auto* attr = worker->AddAttribute(); - attr->SetKey(k); - attr->SetValue(v); - - if (k == NCommonAttrs::JOBID_ATTR) { - worker->SetJobId(v); - } - if (k == NCommonAttrs::OPERATIONID_ATTR) { - worker->SetOperationId(v); - } - } - } - } - - TEvFreeWorkersNotify::TEvFreeWorkersNotify(ui64 resourceId) { - Record.SetResourceId(resourceId); + TEvAllocateWorkersResponse::TEvAllocateWorkersResponse(ui64 resourceId, const TVector<TWorkerInfo::TPtr>& infos) { + auto& group = *Record.MutableNodes(); + group.SetResourceId(resourceId); + for (const auto& workerInfo : infos) { + auto* worker = group.AddWorker(); + *worker->MutableGuid() = GetGuidAsString(workerInfo->WorkerId); // for debug only + worker->SetNodeId(workerInfo->NodeId); + worker->SetClusterName(workerInfo->ClusterName); + + for (const auto& [k, v] : workerInfo->Attributes) { + auto* attr = worker->AddAttribute(); + attr->SetKey(k); + attr->SetValue(v); + + if (k == NCommonAttrs::JOBID_ATTR) { + worker->SetJobId(v); + } + if (k == NCommonAttrs::OPERATIONID_ATTR) { + worker->SetOperationId(v); + } + } + } + } + + TEvFreeWorkersNotify::TEvFreeWorkersNotify(ui64 resourceId) { + Record.SetResourceId(resourceId); } - - TEvRegisterNode::TEvRegisterNode(const Yql::DqsProto::RegisterNodeRequest& request) { - *Record.MutableRequest() = request; - Record.SetIsForwarded(false); - } - - TEvRegisterNodeResponse::TEvRegisterNodeResponse(const TVector<NActors::TEvInterconnect::TNodeInfo>& nodes, ui32 epoch) - { - auto* response = Record.MutableResponse(); - for (const auto& node : nodes) { - auto* n = response->AddNodes(); - n->SetPort(node.Port); - n->SetAddress(node.Address); - n->SetNodeId(node.NodeId); - } - response->SetEpoch(epoch); - } - - TEvJobStop::TEvJobStop(const Yql::DqsProto::JobStopRequest& request) { - *Record.MutableRequest() = request; - Record.SetIsForwarded(false); - } - - TEvOperationStop::TEvOperationStop(const Yql::DqsProto::OperationStopRequest& request) { - *Record.MutableRequest() = request; - Record.SetIsForwarded(false); - } - + + TEvRegisterNode::TEvRegisterNode(const Yql::DqsProto::RegisterNodeRequest& request) { + *Record.MutableRequest() = request; + Record.SetIsForwarded(false); + } + + TEvRegisterNodeResponse::TEvRegisterNodeResponse(const TVector<NActors::TEvInterconnect::TNodeInfo>& nodes, ui32 epoch) + { + auto* response = Record.MutableResponse(); + for (const auto& node : nodes) { + auto* n = response->AddNodes(); + n->SetPort(node.Port); + n->SetAddress(node.Address); + n->SetNodeId(node.NodeId); + } + response->SetEpoch(epoch); + } + + TEvJobStop::TEvJobStop(const Yql::DqsProto::JobStopRequest& request) { + *Record.MutableRequest() = request; + Record.SetIsForwarded(false); + } + + TEvOperationStop::TEvOperationStop(const Yql::DqsProto::OperationStopRequest& request) { + *Record.MutableRequest() = request; + Record.SetIsForwarded(false); + } + TEvIsReady::TEvIsReady(const Yql::DqsProto::IsReadyRequest &request) { *Record.MutableRequest() = request; Record.SetIsForwarded(false); @@ -112,4 +112,4 @@ namespace NYql::NDqs { TEvQueryStatus::TEvQueryStatus(const Yql::DqsProto::QueryStatusRequest& request) { *Record.MutableRequest() = request; } -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/events.h b/ydb/library/yql/providers/dq/worker_manager/interface/events.h index b2dc897f73..a2d0c9f540 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/events.h +++ b/ydb/library/yql/providers/dq/worker_manager/interface/events.h @@ -5,90 +5,90 @@ #include <ydb/library/yql/dq/common/dq_common.h> #include <ydb/library/yql/utils/log/log.h> - + #include <library/cpp/actors/core/actor.h> #include <library/cpp/actors/interconnect/interconnect.h> -#include <util/generic/guid.h> - +#include <util/generic/guid.h> + #include <utility> namespace NYql::NDqs { using TDqResManEvents = NDq::TBaseDqResManEvents<NActors::TEvents::EEventSpace::ES_USERSPACE>; - + struct TEvAllocateWorkersRequest : NActors::TEventPB<TEvAllocateWorkersRequest, NYql::NDqProto::TAllocateWorkersRequest, TDqResManEvents::ES_ALLOCATE_WORKERS_REQUEST> { TEvAllocateWorkersRequest() = default; - - explicit TEvAllocateWorkersRequest( - ui32 count, + + explicit TEvAllocateWorkersRequest( + ui32 count, const TString& user, - const TMaybe<ui64>& globalResourceId = TMaybe<ui64>()); + const TMaybe<ui64>& globalResourceId = TMaybe<ui64>()); }; struct TEvAllocateWorkersResponse : NActors::TEventPB<TEvAllocateWorkersResponse, NYql::NDqProto::TAllocateWorkersResponse, TDqResManEvents::ES_ALLOCATE_WORKERS_RESPONSE> { TEvAllocateWorkersResponse(); - explicit TEvAllocateWorkersResponse(const TString& error, NYql::NDqProto::EErrorCode code = NYql::NDqProto::EUNKNOWN); + explicit TEvAllocateWorkersResponse(const TString& error, NYql::NDqProto::EErrorCode code = NYql::NDqProto::EUNKNOWN); explicit TEvAllocateWorkersResponse(ui64 resourceId, const TVector<NActors::TActorId>& ids); - explicit TEvAllocateWorkersResponse(ui64 resourceId, const TVector<TWorkerInfo::TPtr>& workerInfos); + explicit TEvAllocateWorkersResponse(ui64 resourceId, const TVector<TWorkerInfo::TPtr>& workerInfos); explicit TEvAllocateWorkersResponse(ui64 resourceId, const TVector<ui32>& nodes); }; struct TEvFreeWorkersNotify : NActors::TEventPB<TEvFreeWorkersNotify, NYql::NDqProto::TFreeWorkersNotify, TDqResManEvents::ES_FREE_WORKERS_NOTIFICATION> { TEvFreeWorkersNotify() = default; - explicit TEvFreeWorkersNotify(ui64 id); - }; - - struct TEvRegisterNode - : NActors::TEventPB<TEvRegisterNode, NYql::NDqProto::TEvRegisterNode, TDqResManEvents::ES_REGISTER_NODE> { - - TEvRegisterNode() = default; - TEvRegisterNode(const Yql::DqsProto::RegisterNodeRequest& request); - }; - - struct TEvRegisterNodeResponse - : NActors::TEventPB<TEvRegisterNodeResponse, NYql::NDqProto::TEvRegisterNodeResponse, TDqResManEvents::ES_REGISTER_NODE_RESPONSE> { - TEvRegisterNodeResponse() = default; - - TEvRegisterNodeResponse(const TVector<NActors::TEvInterconnect::TNodeInfo>& nodes, ui32 epoch); - }; - - struct TEvJobStop - : NActors::TEventPB<TEvJobStop, NYql::NDqProto::TEvJobStop, TDqResManEvents::ES_JOB_STOP> { - - TEvJobStop() = default; - TEvJobStop(const Yql::DqsProto::JobStopRequest& request); - }; - - struct TEvOperationStop - : NActors::TEventPB<TEvOperationStop, NYql::NDqProto::TEvOperationStop, TDqResManEvents::ES_OPERATION_STOP> { - - TEvOperationStop() = default; - TEvOperationStop(const Yql::DqsProto::OperationStopRequest& request); - }; - - struct TEvOperationStopResponse - : NActors::TEventPB<TEvOperationStopResponse, NYql::NDqProto::TEvOperationStopResponse, TDqResManEvents::ES_OPERATION_STOP_RESPONSE> { - - TEvOperationStopResponse() = default; - }; - - struct TEvClusterStatus - : NActors::TEventPB<TEvClusterStatus, NYql::NDqProto::TEvClusterStatus, TDqResManEvents::ES_CLUSTER_STATUS> { - - TEvClusterStatus() = default; - }; - - struct TEvClusterStatusResponse - : NActors::TEventPB<TEvClusterStatusResponse, NYql::NDqProto::TEvClusterStatusResponse, TDqResManEvents::ES_CLUSTER_STATUS_RESPONSE> { - - TEvClusterStatusResponse() = default; - }; - + explicit TEvFreeWorkersNotify(ui64 id); + }; + + struct TEvRegisterNode + : NActors::TEventPB<TEvRegisterNode, NYql::NDqProto::TEvRegisterNode, TDqResManEvents::ES_REGISTER_NODE> { + + TEvRegisterNode() = default; + TEvRegisterNode(const Yql::DqsProto::RegisterNodeRequest& request); + }; + + struct TEvRegisterNodeResponse + : NActors::TEventPB<TEvRegisterNodeResponse, NYql::NDqProto::TEvRegisterNodeResponse, TDqResManEvents::ES_REGISTER_NODE_RESPONSE> { + TEvRegisterNodeResponse() = default; + + TEvRegisterNodeResponse(const TVector<NActors::TEvInterconnect::TNodeInfo>& nodes, ui32 epoch); + }; + + struct TEvJobStop + : NActors::TEventPB<TEvJobStop, NYql::NDqProto::TEvJobStop, TDqResManEvents::ES_JOB_STOP> { + + TEvJobStop() = default; + TEvJobStop(const Yql::DqsProto::JobStopRequest& request); + }; + + struct TEvOperationStop + : NActors::TEventPB<TEvOperationStop, NYql::NDqProto::TEvOperationStop, TDqResManEvents::ES_OPERATION_STOP> { + + TEvOperationStop() = default; + TEvOperationStop(const Yql::DqsProto::OperationStopRequest& request); + }; + + struct TEvOperationStopResponse + : NActors::TEventPB<TEvOperationStopResponse, NYql::NDqProto::TEvOperationStopResponse, TDqResManEvents::ES_OPERATION_STOP_RESPONSE> { + + TEvOperationStopResponse() = default; + }; + + struct TEvClusterStatus + : NActors::TEventPB<TEvClusterStatus, NYql::NDqProto::TEvClusterStatus, TDqResManEvents::ES_CLUSTER_STATUS> { + + TEvClusterStatus() = default; + }; + + struct TEvClusterStatusResponse + : NActors::TEventPB<TEvClusterStatusResponse, NYql::NDqProto::TEvClusterStatusResponse, TDqResManEvents::ES_CLUSTER_STATUS_RESPONSE> { + + TEvClusterStatusResponse() = default; + }; + struct TEvQueryStatus : NActors::TEventPB<TEvQueryStatus, NYql::NDqProto::TEvQueryStatus, TDqResManEvents::ES_QUERY_STATUS> { @@ -115,18 +115,18 @@ using TDqResManEvents = NDq::TBaseDqResManEvents<NActors::TEvents::EEventSpace:: TEvIsReadyResponse() = default; }; - struct TEvRoutesRequest - : NActors::TEventPB<TEvRoutesRequest, NYql::NDqProto::TEvRoutesRequest, TDqResManEvents::ES_ROUTES> { - - TEvRoutesRequest() = default; - }; - - struct TEvRoutesResponse - : NActors::TEventPB<TEvRoutesResponse, NYql::NDqProto::TEvRoutesResponse, TDqResManEvents::ES_ROUTES_RESPONSE> { - - TEvRoutesResponse() = default; - }; - + struct TEvRoutesRequest + : NActors::TEventPB<TEvRoutesRequest, NYql::NDqProto::TEvRoutesRequest, TDqResManEvents::ES_ROUTES> { + + TEvRoutesRequest() = default; + }; + + struct TEvRoutesResponse + : NActors::TEventPB<TEvRoutesResponse, NYql::NDqProto::TEvRoutesResponse, TDqResManEvents::ES_ROUTES_RESPONSE> { + + TEvRoutesResponse() = default; + }; + struct TEvGetMasterRequest : NActors::TEventPB<TEvGetMasterRequest, NYql::NDqProto::TEvGetMasterRequest, TDqResManEvents::ES_GET_MASTER> { diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.cpp b/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.cpp index e177ccfcc0..634b25fb07 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.cpp +++ b/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.cpp @@ -30,34 +30,34 @@ namespace NYql::NDqs { } TWorkerInfo::TWorkerInfo( - const TString& startTime, + const TString& startTime, ui32 nodeId, TGUID workerId, const Yql::DqsProto::RegisterNodeRequest& request, NYql::TSensorsGroupPtr metrics, - const TInflightLimiter::TPtr& inflightLimiter, - TGlobalResources& globalResources) + const TInflightLimiter::TPtr& inflightLimiter, + TGlobalResources& globalResources) : NodeId(nodeId) , WorkerId(workerId) , LastPingTime(TInstant::Now()) , Revision(request.GetRevision()) , ClusterName(request.GetClusterName()) , Address(request.GetAddress()) - , StartTime(!request.GetStartTime().empty() - ? request.GetStartTime() - : startTime) + , StartTime(!request.GetStartTime().empty() + ? request.GetStartTime() + : startTime) , Port(request.GetPort()) , Epoch(request.GetEpoch()) , Capabilities(request.GetCapabilities()) - , Capacity(request.GetCapacity()?request.GetCapacity():1) + , Capacity(request.GetCapacity()?request.GetCapacity():1) , Metrics(std::move(metrics)) , InflightLimiter(inflightLimiter) - , ClusterResources(TClusterResources(globalResources, ClusterName)) + , ClusterResources(TClusterResources(globalResources, ClusterName)) { -#define ADD_METRIC(name) \ - name = Metrics->GetCounter(#name) -#define ADD_METRIC_DERIV(name) \ - name = Metrics->GetCounter(#name, /*derivative=*/ true) +#define ADD_METRIC(name) \ + name = Metrics->GetCounter(#name) +#define ADD_METRIC_DERIV(name) \ + name = Metrics->GetCounter(#name, /*derivative=*/ true) ADD_METRIC(CurrentDownloadsSum); ADD_METRIC(CurrentDownloadsMax); @@ -76,89 +76,89 @@ namespace NYql::NDqs { ADD_METRIC(UsedDiskSizeMax); ADD_METRIC(UsedDiskSizeArgMax); - ADD_METRIC(ActiveDownloadsSum); - ADD_METRIC(DeadWorkers); - - ADD_METRIC(CpuTotalSum); - ADD_METRIC(MajorPageFaultsSum); - - ADD_METRIC_DERIV(FileAddCounter); - ADD_METRIC_DERIV(FileRemoveCounter); - - ADD_METRIC(RunningActors); - + ADD_METRIC(ActiveDownloadsSum); + ADD_METRIC(DeadWorkers); + + ADD_METRIC(CpuTotalSum); + ADD_METRIC(MajorPageFaultsSum); + + ADD_METRIC_DERIV(FileAddCounter); + ADD_METRIC_DERIV(FileRemoveCounter); + + ADD_METRIC(RunningActors); + #undef ADD_METRIC -#undef ADD_METRIC_DERIV +#undef ADD_METRIC_DERIV for (const auto& attr : request.GetAttribute()) { Attributes[attr.GetKey()] = attr.GetValue(); } - ClusterResources.AddCapacity(Capacity); - + ClusterResources.AddCapacity(Capacity); + Update(request); } bool TWorkerInfo::Update(const Yql::DqsProto::RegisterNodeRequest& request) { bool needResume = false; - auto now = TInstant::Now(); - TDuration interval = now-LastPingTime; - LastPingTime = now; - - if (request.GetZombie()) { - OnDead(); - return false; - } - - auto curStime = TDuration::MicroSeconds(request.GetRusage().GetStime()); - auto curUtime = TDuration::MicroSeconds(request.GetRusage().GetUtime()); - - i64 cpuTotalCur = 0; - i64 cpuSystemCur = 0; - i64 cpuUserCur = 0; - - if (interval > TDuration::Seconds(1)) { - cpuTotalCur = 100*(curStime+curUtime-Stime-Utime).MicroSeconds(); - cpuTotalCur /= interval.MicroSeconds(); - - cpuSystemCur = 100*(curStime-Stime).MicroSeconds(); - cpuSystemCur /= interval.MicroSeconds(); - - cpuUserCur = 100*(curUtime-Utime).MicroSeconds(); - cpuUserCur /= interval.MicroSeconds(); - } - - Stime = curStime; - Utime = curUtime; - - auto cpuTotalDelta = (cpuTotalCur - CpuTotal); - CpuTotal = cpuTotalCur; - *CpuTotalSum += cpuTotalDelta; - - CpuSystem = cpuSystemCur; - CpuUser = cpuUserCur; - - auto pageFaultsDelta = (request.GetRusage().GetMajorPageFaults() - MajorPageFaults); - MajorPageFaults = request.GetRusage().GetMajorPageFaults(); - *MajorPageFaultsSum += pageFaultsDelta; - + auto now = TInstant::Now(); + TDuration interval = now-LastPingTime; + LastPingTime = now; + + if (request.GetZombie()) { + OnDead(); + return false; + } + + auto curStime = TDuration::MicroSeconds(request.GetRusage().GetStime()); + auto curUtime = TDuration::MicroSeconds(request.GetRusage().GetUtime()); + + i64 cpuTotalCur = 0; + i64 cpuSystemCur = 0; + i64 cpuUserCur = 0; + + if (interval > TDuration::Seconds(1)) { + cpuTotalCur = 100*(curStime+curUtime-Stime-Utime).MicroSeconds(); + cpuTotalCur /= interval.MicroSeconds(); + + cpuSystemCur = 100*(curStime-Stime).MicroSeconds(); + cpuSystemCur /= interval.MicroSeconds(); + + cpuUserCur = 100*(curUtime-Utime).MicroSeconds(); + cpuUserCur /= interval.MicroSeconds(); + } + + Stime = curStime; + Utime = curUtime; + + auto cpuTotalDelta = (cpuTotalCur - CpuTotal); + CpuTotal = cpuTotalCur; + *CpuTotalSum += cpuTotalDelta; + + CpuSystem = cpuSystemCur; + CpuUser = cpuUserCur; + + auto pageFaultsDelta = (request.GetRusage().GetMajorPageFaults() - MajorPageFaults); + MajorPageFaults = request.GetRusage().GetMajorPageFaults(); + *MajorPageFaultsSum += pageFaultsDelta; + for (auto file : request.GetFilesOnNode()) { if (Resources.insert(file.GetObjectId()).second) { needResume = true; *FilesCountSum += 1; - *FileAddCounter += 1; + *FileAddCounter += 1; } } - Operations.clear(); - for (const auto& op: request.GetRunningOperation()) { - Operations.insert(op); - } + Operations.clear(); + for (const auto& op: request.GetRunningOperation()) { + Operations.insert(op); + } + + int actorsDelta = request.GetRunningWorkers()-RunningWorkerActors; + *RunningActors += actorsDelta; - int actorsDelta = request.GetRunningWorkers()-RunningWorkerActors; - *RunningActors += actorsDelta; - - RunningWorkerActors = request.GetRunningWorkers(); + RunningWorkerActors = request.GetRunningWorkers(); if (static_cast<int>(Resources.size()) > static_cast<int>(request.GetFilesOnNode().size())) { needResume = true; @@ -177,22 +177,22 @@ namespace NYql::NDqs { YQL_LOG(DEBUG) << "Remove resource " << k << " from worker " << GetGuidAsString(WorkerId); Resources.erase(k); } - *FileRemoveCounter += toDrop.size(); + *FileRemoveCounter += toDrop.size(); *FilesCountSum -= toDrop.size(); } // ordered as in LRU cache ResourcesOrdered = std::move(request.GetFilesOnNode()); - if (*FreeDiskSizeArgMin == 0 || *FreeDiskSizeMin > request.GetFreeDiskSize()) { + if (*FreeDiskSizeArgMin == 0 || *FreeDiskSizeMin > request.GetFreeDiskSize()) { *FreeDiskSizeMin = request.GetFreeDiskSize(); *FreeDiskSizeArgMin = NodeId; } - if (*FreeDiskSizeArgMin == NodeId && *FreeDiskSizeMin != request.GetFreeDiskSize()) { - *FreeDiskSizeMin = request.GetFreeDiskSize(); - } - + if (*FreeDiskSizeArgMin == NodeId && *FreeDiskSizeMin != request.GetFreeDiskSize()) { + *FreeDiskSizeMin = request.GetFreeDiskSize(); + } + *FreeDiskSizeSum += (request.GetFreeDiskSize() - FreeDiskSize); FreeDiskSize = request.GetFreeDiskSize(); @@ -224,20 +224,20 @@ namespace NYql::NDqs { void TWorkerInfo::RemoveFromDownloadList(const TString& objectId) { auto delta = DownloadList.erase(objectId); if (delta) { - InflightLimiter->MarkDownloadFinished(objectId); - *ActiveDownloadsSum -= ActiveDownloads.erase(objectId); + InflightLimiter->MarkDownloadFinished(objectId); + *ActiveDownloadsSum -= ActiveDownloads.erase(objectId); } *CurrentDownloadsSum -= delta; } void TWorkerInfo::AddToDownloadList(const THashMap<TString, TFileResource>& downloadList) { - for (const auto& [k, v] : downloadList) { - AddToDownloadList(k, v); - } + for (const auto& [k, v] : downloadList) { + AddToDownloadList(k, v); + } } bool TWorkerInfo::AddToDownloadList(const TString& key, const TFileResource& value) { - if (!Resources.contains(key) && DownloadList.insert({key, value}).second) { + if (!Resources.contains(key) && DownloadList.insert({key, value}).second) { *CurrentDownloadsSum += 1; return true; } @@ -251,13 +251,13 @@ namespace NYql::NDqs { THashMap<TString, TWorkerInfo::TFileResource> TWorkerInfo::GetResourcesForDownloading() { for (const auto& it : DownloadList) { TString id = it.first; - if (InflightLimiter->CanDownload(id) && !ActiveDownloads.contains(id)) { - ActiveDownloads.emplace(id, it.second); - *ActiveDownloadsSum += 1; - InflightLimiter->MarkDownloadStarted(id); + if (InflightLimiter->CanDownload(id) && !ActiveDownloads.contains(id)) { + ActiveDownloads.emplace(id, it.second); + *ActiveDownloadsSum += 1; + InflightLimiter->MarkDownloadStarted(id); } } - return ActiveDownloads; + return ActiveDownloads; } const THashSet<TString>& TWorkerInfo::GetResources() { @@ -265,54 +265,54 @@ namespace NYql::NDqs { } void TWorkerInfo::OnDead() { - if (IsDead) { - return; - } + if (IsDead) { + return; + } IsDead = true; - for (const auto& [id, _] : ActiveDownloads) { - InflightLimiter->MarkDownloadFinished(id); + for (const auto& [id, _] : ActiveDownloads) { + InflightLimiter->MarkDownloadFinished(id); } - *ActiveDownloadsSum -= ActiveDownloads.size(); - ActiveDownloads.clear(); + *ActiveDownloadsSum -= ActiveDownloads.size(); + ActiveDownloads.clear(); *CurrentDownloadsSum -= DownloadList.size(); *FilesCountSum -= Resources.size(); *UsedDiskSizeSum += - UsedDiskSize; *FreeDiskSizeSum += - FreeDiskSize; - - *DeadWorkers += 1; - - ClusterResources.AddCapacity(-Capacity); - ClusterResources.AddRunning(-RunningRequests); - RunningRequests = 0; - - *CpuTotalSum -= CpuTotal; - *MajorPageFaultsSum -= MajorPageFaults; - - *RunningActors -= RunningWorkerActors; - Operations.clear(); - } - - TWorkerInfo::~TWorkerInfo() { - OnDead(); + + *DeadWorkers += 1; + + ClusterResources.AddCapacity(-Capacity); + ClusterResources.AddRunning(-RunningRequests); + RunningRequests = 0; + + *CpuTotalSum -= CpuTotal; + *MajorPageFaultsSum -= MajorPageFaults; + + *RunningActors -= RunningWorkerActors; + Operations.clear(); } - - bool TWorkerInfo::Acquire() { - if (IsDead) { - return true; - } - ClusterResources.AddRunning(1); - return ++RunningRequests >= Capacity; - } - - bool TWorkerInfo::Release() { - if (IsDead) { - return false; - } - UseTime += TInstant::Now() - RequestStartTime; - ClusterResources.AddRunning(-1); - return --RunningRequests < Capacity; - } - -} // namespace NYql::NDqs + + TWorkerInfo::~TWorkerInfo() { + OnDead(); + } + + bool TWorkerInfo::Acquire() { + if (IsDead) { + return true; + } + ClusterResources.AddRunning(1); + return ++RunningRequests >= Capacity; + } + + bool TWorkerInfo::Release() { + if (IsDead) { + return false; + } + UseTime += TInstant::Now() - RequestStartTime; + ClusterResources.AddRunning(-1); + return --RunningRequests < Capacity; + } + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h b/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h index 009a74b267..5281a844f4 100644 --- a/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h +++ b/ydb/library/yql/providers/dq/worker_manager/interface/worker_info.h @@ -11,10 +11,10 @@ namespace NYql::NDqs { -class TInflightLimiter: public TThrRefBase { +class TInflightLimiter: public TThrRefBase { public: - using TPtr = TIntrusivePtr<TInflightLimiter>; - + using TPtr = TIntrusivePtr<TInflightLimiter>; + TInflightLimiter(i32 inflightLimit); bool CanDownload(const TString& id); void MarkDownloadStarted(const TString& id); @@ -25,76 +25,76 @@ private: i32 InflightLimit; }; -struct TGlobalResources { - TGlobalResources(TSensorsGroupPtr metrics) - : Metrics(metrics) - , Capacity(0) - , RunningRequests(0) - { } - - void AddCapacity(int value) { - Capacity += value; - } - - void AddRunning(int value) { - RunningRequests += value; - } - - i64 GetCapacity() const { - return Capacity; - } - - i64 GetRunningRequests() const { - return RunningRequests; - } - - TSensorsGroupPtr GetMetrics() { - return Metrics; - } - -private: - TSensorsGroupPtr Metrics; - i64 Capacity; - i64 RunningRequests; - - TGlobalResources(); - TGlobalResources(const TGlobalResources&); -}; - -struct TClusterResources { -public: - TClusterResources(TGlobalResources& resources, const TString& clusterName) - : Resources(resources) - , Capacity(Resources.GetMetrics() - ->GetSubgroup("component", "lists") - ->GetSubgroup("ClusterName", clusterName) - ->GetCounter("Capacity")) - , RunningRequests(Resources.GetMetrics() - ->GetSubgroup("component", "lists") - ->GetSubgroup("ClusterName", clusterName) - ->GetCounter("RunningRequests")) - { } - - void Reset() { } - - void AddCapacity(int value) { - Resources.AddCapacity(value); - *Capacity += value; - } - - void AddRunning(int value) { - Resources.AddRunning(value); - *RunningRequests += value; - } - -private: - TGlobalResources& Resources; - NMonitoring::TDynamicCounters::TCounterPtr Capacity; - NMonitoring::TDynamicCounters::TCounterPtr RunningRequests; -}; - -struct TWorkerInfo: public TThrRefBase { - using TPtr = TIntrusivePtr<TWorkerInfo>; +struct TGlobalResources { + TGlobalResources(TSensorsGroupPtr metrics) + : Metrics(metrics) + , Capacity(0) + , RunningRequests(0) + { } + + void AddCapacity(int value) { + Capacity += value; + } + + void AddRunning(int value) { + RunningRequests += value; + } + + i64 GetCapacity() const { + return Capacity; + } + + i64 GetRunningRequests() const { + return RunningRequests; + } + + TSensorsGroupPtr GetMetrics() { + return Metrics; + } + +private: + TSensorsGroupPtr Metrics; + i64 Capacity; + i64 RunningRequests; + + TGlobalResources(); + TGlobalResources(const TGlobalResources&); +}; + +struct TClusterResources { +public: + TClusterResources(TGlobalResources& resources, const TString& clusterName) + : Resources(resources) + , Capacity(Resources.GetMetrics() + ->GetSubgroup("component", "lists") + ->GetSubgroup("ClusterName", clusterName) + ->GetCounter("Capacity")) + , RunningRequests(Resources.GetMetrics() + ->GetSubgroup("component", "lists") + ->GetSubgroup("ClusterName", clusterName) + ->GetCounter("RunningRequests")) + { } + + void Reset() { } + + void AddCapacity(int value) { + Resources.AddCapacity(value); + *Capacity += value; + } + + void AddRunning(int value) { + Resources.AddRunning(value); + *RunningRequests += value; + } + +private: + TGlobalResources& Resources; + NMonitoring::TDynamicCounters::TCounterPtr Capacity; + NMonitoring::TDynamicCounters::TCounterPtr RunningRequests; +}; + +struct TWorkerInfo: public TThrRefBase { + using TPtr = TIntrusivePtr<TWorkerInfo>; using TFileResource = Yql::DqsProto::TFile; const ui32 NodeId; @@ -103,14 +103,14 @@ struct TWorkerInfo: public TThrRefBase { const TString Revision; const TString ClusterName; const TString Address; - const TString StartTime; + const TString StartTime; const ui32 Port; ui32 Epoch; THashMap<TString, TString> Attributes; - THashSet<TString> Operations; + THashSet<TString> Operations; ui64 UseCount = 0; - TInstant RequestStartTime; + TInstant RequestStartTime; TDuration UseTime; bool IsDead = false; bool Stopping = false; @@ -119,27 +119,27 @@ struct TWorkerInfo: public TThrRefBase { i64 FreeDiskSize = 0; i64 UsedDiskSize = 0; - const int Capacity; - const int CpuCores = 1; // unused yet - - TDuration Stime; - TDuration Utime; - i64 CpuSystem = 0; - i64 CpuUser = 0; - i64 CpuTotal = 0; - i64 MajorPageFaults = 0; - - int RunningRequests = 0; + const int Capacity; + const int CpuCores = 1; // unused yet + + TDuration Stime; + TDuration Utime; + i64 CpuSystem = 0; + i64 CpuUser = 0; + i64 CpuTotal = 0; + i64 MajorPageFaults = 0; + + int RunningRequests = 0; int RunningWorkerActors = 0; TWorkerInfo( - const TString& startTime, + const TString& startTime, ui32 nodeId, TGUID workerId, const Yql::DqsProto::RegisterNodeRequest& request, NYql::TSensorsGroupPtr metrics, - const TInflightLimiter::TPtr& inflightLimiter, - TGlobalResources& globalResources + const TInflightLimiter::TPtr& inflightLimiter, + TGlobalResources& globalResources ); ~TWorkerInfo(); @@ -162,33 +162,33 @@ struct TWorkerInfo: public TThrRefBase { return ResourcesOrdered; } - const auto& GetActiveDownloads() { - return ActiveDownloads; - } - + const auto& GetActiveDownloads() { + return ActiveDownloads; + } + void OnDead(); - bool Acquire(); - - bool Release(); - + bool Acquire(); + + bool Release(); + private: THashSet<TString> Resources; google::protobuf::RepeatedPtrField<::Yql::DqsProto::RegisterNodeRequest_LocalFile> ResourcesOrdered; THashMap<TString, TFileResource> DownloadList; TSensorsGroupPtr Metrics; - TInflightLimiter::TPtr InflightLimiter; - TClusterResources ClusterResources; - THashMap<TString, TFileResource> ActiveDownloads; + TInflightLimiter::TPtr InflightLimiter; + TClusterResources ClusterResources; + THashMap<TString, TFileResource> ActiveDownloads; TSensorCounterPtr CurrentDownloadsSum; TSensorCounterPtr CurrentDownloadsMax; TSensorCounterPtr CurrentDownloadsArgMax; - TSensorCounterPtr ActiveDownloadsSum; - - TSensorCounterPtr DeadWorkers; - + TSensorCounterPtr ActiveDownloadsSum; + + TSensorCounterPtr DeadWorkers; + TSensorCounterPtr FilesCountSum; TSensorCounterPtr FilesCountMax; TSensorCounterPtr FilesCountArgMax; @@ -200,28 +200,28 @@ private: TSensorCounterPtr UsedDiskSizeSum; TSensorCounterPtr UsedDiskSizeMax; TSensorCounterPtr UsedDiskSizeArgMax; - - TSensorCounterPtr CpuTotalSum; - TSensorCounterPtr MajorPageFaultsSum; - - TSensorCounterPtr FileRemoveCounter; - TSensorCounterPtr FileAddCounter; - - TSensorCounterPtr RunningActors; + + TSensorCounterPtr CpuTotalSum; + TSensorCounterPtr MajorPageFaultsSum; + + TSensorCounterPtr FileRemoveCounter; + TSensorCounterPtr FileAddCounter; + + TSensorCounterPtr RunningActors; +}; + +struct TWorkerInfoPtrComparator { + bool operator()(const TWorkerInfo::TPtr& a, const TWorkerInfo::TPtr& b) const { + auto scoreA = (a->CpuTotal+1) * (a->RunningRequests + a->RunningWorkerActors + 1); + auto scoreB = (b->CpuTotal+1) * (b->RunningRequests + b->RunningWorkerActors + 1); + if (scoreA < scoreB) { + return true; + } else if (scoreA > scoreB) { + return false; + } else { + return a.Get() < b.Get(); + } + } }; -struct TWorkerInfoPtrComparator { - bool operator()(const TWorkerInfo::TPtr& a, const TWorkerInfo::TPtr& b) const { - auto scoreA = (a->CpuTotal+1) * (a->RunningRequests + a->RunningWorkerActors + 1); - auto scoreB = (b->CpuTotal+1) * (b->RunningRequests + b->RunningWorkerActors + 1); - if (scoreA < scoreB) { - return true; - } else if (scoreA > scoreB) { - return false; - } else { - return a.Get() < b.Get(); - } - } -}; - -} // namespace NYql::NDqs +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp index bc754b45c7..5ecf0a4a85 100644 --- a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp +++ b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp @@ -10,47 +10,47 @@ #include <ydb/library/yql/utils/failure_injector/failure_injector.h> #include <ydb/library/yql/utils/log/log.h> - + #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/events.h> #include <library/cpp/actors/interconnect/interconnect.h> -#include "worker_manager_common.h" - +#include "worker_manager_common.h" + #include <util/generic/vector.h> #include <util/system/mutex.h> -#include <util/random/random.h> -#include <util/system/rusage.h> +#include <util/random/random.h> +#include <util/system/rusage.h> + +using namespace NActors; -using namespace NActors; - namespace NYql::NDqs { - -union TDqLocalResourceId { - struct { - ui32 Counter; - ui16 Seed; - ui16 NodeId; - }; - ui64 Data; -}; - -static_assert(sizeof(TDqLocalResourceId) == 8); - -class TLocalWorkerManager: public TWorkerManagerCommon<TLocalWorkerManager> { - -public: + +union TDqLocalResourceId { + struct { + ui32 Counter; + ui16 Seed; + ui16 NodeId; + }; + ui64 Data; +}; + +static_assert(sizeof(TDqLocalResourceId) == 8); + +class TLocalWorkerManager: public TWorkerManagerCommon<TLocalWorkerManager> { + +public: static constexpr char ActorName[] = "YQL_DQ_LWM"; - - TLocalWorkerManager(const TLocalWorkerManagerOptions& options) - : TWorkerManagerCommon<TLocalWorkerManager>(&TLocalWorkerManager::Handler) - , Options(options) - , MemoryQuoter(std::make_shared<NDq::TResourceQuoter>(Options.MkqlTotalMemoryLimit)) + + TLocalWorkerManager(const TLocalWorkerManagerOptions& options) + : TWorkerManagerCommon<TLocalWorkerManager>(&TLocalWorkerManager::Handler) + , Options(options) + , MemoryQuoter(std::make_shared<NDq::TResourceQuoter>(Options.MkqlTotalMemoryLimit)) { - Options.Counters.MkqlMemoryLimit->Set(Options.MkqlTotalMemoryLimit); - Options.Counters.MkqlMemoryAllocated->Set(0); + Options.Counters.MkqlMemoryLimit->Set(Options.MkqlTotalMemoryLimit); + Options.Counters.MkqlMemoryAllocated->Set(0); - MemoryQuoter->SetNotifier([limitCounter = Options.Counters.MkqlMemoryLimit, allocatedCounter = Options.Counters.MkqlMemoryAllocated](const ui64 limit, ui64 allocated) { + MemoryQuoter->SetNotifier([limitCounter = Options.Counters.MkqlMemoryLimit, allocatedCounter = Options.Counters.MkqlMemoryAllocated](const ui64 limit, ui64 allocated) { limitCounter->Set(limit); allocatedCounter->Set(allocated); }); @@ -66,72 +66,72 @@ public: }; } -private: - STRICT_STFUNC(Handler, { +private: + STRICT_STFUNC(Handler, { hFunc(TEvAllocateWorkersRequest, OnAllocateWorkersRequest) hFunc(TEvFreeWorkersNotify, OnFreeWorkers) - cFunc(TEvents::TEvPoison::EventType, PassAway) - cFunc(TEvents::TEvBootstrap::EventType, Bootstrap) - cFunc(TEvents::TEvWakeup::EventType, WakeUp) - IgnoreFunc(TEvInterconnect::TEvNodeConnected) + cFunc(TEvents::TEvPoison::EventType, PassAway) + cFunc(TEvents::TEvBootstrap::EventType, Bootstrap) + cFunc(TEvents::TEvWakeup::EventType, WakeUp) + IgnoreFunc(TEvInterconnect::TEvNodeConnected) hFunc(TEvInterconnect::TEvNodeDisconnected, OnDisconnected) hFunc(TEvents::TEvUndelivered, OnUndelivered) hFunc(TEvConfigureFailureInjectorRequest, OnConfigureFailureInjector) - HFunc(TEvRoutesRequest, OnRoutesRequest) + HFunc(TEvRoutesRequest, OnRoutesRequest) hFunc(TEvQueryStatus, OnQueryStatus) - }) - + }) + TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override { - return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0); - } - - void Bootstrap() { - ResourceId.Seed = static_cast<ui16>(RandomNumber<ui64>()); - ResourceId.Counter = 0; - - Send(SelfId(), new TEvents::TEvWakeup()); - } - - void WakeUp() { - auto currentRusage = TRusage::Get(); - TRusage delta; - delta.Utime = currentRusage.Utime - Rusage.Utime; - delta.Stime = currentRusage.Stime - Rusage.Stime; - delta.MajorPageFaults = currentRusage.MajorPageFaults - Rusage.MajorPageFaults; - if (Options.RuntimeData) { - Options.RuntimeData->AddRusageDelta(delta); - } - Rusage = currentRusage; - - FreeOnDeadline(); - - TActivationContext::Schedule(TDuration::MilliSeconds(800), new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup(), 0)); - } - - void DoPassAway() override { - for (const auto& [resourceId, _] : AllocatedWorkers) { - FreeGroup(resourceId); + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap(), 0); + } + + void Bootstrap() { + ResourceId.Seed = static_cast<ui16>(RandomNumber<ui64>()); + ResourceId.Counter = 0; + + Send(SelfId(), new TEvents::TEvWakeup()); + } + + void WakeUp() { + auto currentRusage = TRusage::Get(); + TRusage delta; + delta.Utime = currentRusage.Utime - Rusage.Utime; + delta.Stime = currentRusage.Stime - Rusage.Stime; + delta.MajorPageFaults = currentRusage.MajorPageFaults - Rusage.MajorPageFaults; + if (Options.RuntimeData) { + Options.RuntimeData->AddRusageDelta(delta); + } + Rusage = currentRusage; + + FreeOnDeadline(); + + TActivationContext::Schedule(TDuration::MilliSeconds(800), new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup(), 0)); + } + + void DoPassAway() override { + for (const auto& [resourceId, _] : AllocatedWorkers) { + FreeGroup(resourceId); } AllocatedWorkers.clear(); - _exit(0); + _exit(0); } - void Deallocate(ui32 nodeId) { - TVector<ui64> toDeallocate; - - YQL_LOG(DEBUG) << "Deallocate " << nodeId; - for (const auto& [k, v] : AllocatedWorkers) { + void Deallocate(ui32 nodeId) { + TVector<ui64> toDeallocate; + + YQL_LOG(DEBUG) << "Deallocate " << nodeId; + for (const auto& [k, v] : AllocatedWorkers) { if (v.Sender.NodeId() == nodeId) { - toDeallocate.push_back(k); - } - } - - for (const auto& k : toDeallocate) { - FreeGroup(k); - } - } - + toDeallocate.push_back(k); + } + } + + for (const auto& k : toDeallocate) { + FreeGroup(k); + } + } + void Deallocate(const NActors::TActorId& senderId) { TVector<ui64> toDeallocate; @@ -148,17 +148,17 @@ private: } void OnDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) - { - YQL_LOG(DEBUG) << "Disconnected " << ev->Get()->NodeId; - Unsubscribe(ev->Get()->NodeId); - Deallocate(ev->Get()->NodeId); - } - + { + YQL_LOG(DEBUG) << "Disconnected " << ev->Get()->NodeId; + Unsubscribe(ev->Get()->NodeId); + Deallocate(ev->Get()->NodeId); + } + void OnUndelivered(TEvents::TEvUndelivered::TPtr& ev) - { - Y_VERIFY(ev->Get()->Reason == TEvents::TEvUndelivered::Disconnected - || ev->Get()->Reason == TEvents::TEvUndelivered::ReasonActorUnknown); - + { + Y_VERIFY(ev->Get()->Reason == TEvents::TEvUndelivered::Disconnected + || ev->Get()->Reason == TEvents::TEvUndelivered::ReasonActorUnknown); + YQL_LOG(DEBUG) << "Undelivered " << ev->Sender; switch (ev->Get()->Reason) { @@ -171,8 +171,8 @@ private: default: break; } - } - + } + void OnConfigureFailureInjector(TEvConfigureFailureInjectorRequest::TPtr& ev) { YQL_LOG(DEBUG) << "TEvConfigureFailureInjectorRequest "; @@ -190,108 +190,108 @@ private: } void OnAllocateWorkersRequest(TEvAllocateWorkersRequest::TPtr& ev) { - ui64 resourceId; - if (ev->Get()->Record.GetResourceId()) { - resourceId = ev->Get()->Record.GetResourceId(); - } else { - auto resource = ResourceId; - resource.NodeId = ev->Sender.NodeId(); - resourceId = resource.Data; - ResourceId.Counter ++; - } - bool createComputeActor = ev->Get()->Record.GetCreateComputeActor(); - TString computeActorType = ev->Get()->Record.GetComputeActorType(); - - if (createComputeActor && !Options.CanUseComputeActor) { - Send(ev->Sender, MakeHolder<TEvAllocateWorkersResponse>("Compute Actor Disabled"), 0, ev->Cookie); - return; - } - - YQL_LOG_CTX_SCOPE(ev->Get()->Record.GetTraceId()); - YQL_LOG(DEBUG) << "TLocalWorkerManager::TEvAllocateWorkersRequest " << resourceId; + ui64 resourceId; + if (ev->Get()->Record.GetResourceId()) { + resourceId = ev->Get()->Record.GetResourceId(); + } else { + auto resource = ResourceId; + resource.NodeId = ev->Sender.NodeId(); + resourceId = resource.Data; + ResourceId.Counter ++; + } + bool createComputeActor = ev->Get()->Record.GetCreateComputeActor(); + TString computeActorType = ev->Get()->Record.GetComputeActorType(); + + if (createComputeActor && !Options.CanUseComputeActor) { + Send(ev->Sender, MakeHolder<TEvAllocateWorkersResponse>("Compute Actor Disabled"), 0, ev->Cookie); + return; + } + + YQL_LOG_CTX_SCOPE(ev->Get()->Record.GetTraceId()); + YQL_LOG(DEBUG) << "TLocalWorkerManager::TEvAllocateWorkersRequest " << resourceId; TFailureInjector::Reach("allocate_workers_failure", [] { ::_exit(1); }); - - auto& allocationInfo = AllocatedWorkers[resourceId]; + + auto& allocationInfo = AllocatedWorkers[resourceId]; auto traceId = ev->Get()->Record.GetTraceId(); allocationInfo.TxId = traceId; auto count = ev->Get()->Record.GetCount(); - - Y_VERIFY(count > 0); - - bool canAllocate = MemoryQuoter->Allocate(traceId, 0, count * Options.MkqlInitialMemoryLimit); + + Y_VERIFY(count > 0); + + bool canAllocate = MemoryQuoter->Allocate(traceId, 0, count * Options.MkqlInitialMemoryLimit); if (!canAllocate) { Send(ev->Sender, MakeHolder<TEvAllocateWorkersResponse>("Not enough memory to allocate tasks"), 0, ev->Cookie); return; } - if (allocationInfo.WorkerActors.empty()) { - allocationInfo.WorkerActors.reserve(count); + if (allocationInfo.WorkerActors.empty()) { + allocationInfo.WorkerActors.reserve(count); allocationInfo.Sender = ev->Sender; - if (ev->Get()->Record.GetFreeWorkerAfterMs()) { - allocationInfo.Deadline = - TInstant::Now() + TDuration::MilliSeconds(ev->Get()->Record.GetFreeWorkerAfterMs()); - } - - auto& tasks = *ev->Get()->Record.MutableTask(); - - if (createComputeActor) { - Y_VERIFY(static_cast<int>(tasks.size()) == static_cast<int>(count)); - } - auto resultId = ActorIdFromProto(ev->Get()->Record.GetResultActorId()); - - for (ui32 i = 0; i < count; i++) { - THolder<NActors::IActor> actor; - - if (createComputeActor) { - YQL_LOG(DEBUG) << "Create compute actor: " << computeActorType; - actor.Reset( - NYql::CreateComputeActor( - Options, - Options.MkqlTotalMemoryLimit ? AllocateMemoryFn : nullptr, - Options.MkqlTotalMemoryLimit ? FreeMemoryFn : nullptr, - resultId, - traceId, - std::move(tasks[i]), - computeActorType, - Options.TaskRunnerActorFactory)); - } else { + if (ev->Get()->Record.GetFreeWorkerAfterMs()) { + allocationInfo.Deadline = + TInstant::Now() + TDuration::MilliSeconds(ev->Get()->Record.GetFreeWorkerAfterMs()); + } + + auto& tasks = *ev->Get()->Record.MutableTask(); + + if (createComputeActor) { + Y_VERIFY(static_cast<int>(tasks.size()) == static_cast<int>(count)); + } + auto resultId = ActorIdFromProto(ev->Get()->Record.GetResultActorId()); + + for (ui32 i = 0; i < count; i++) { + THolder<NActors::IActor> actor; + + if (createComputeActor) { + YQL_LOG(DEBUG) << "Create compute actor: " << computeActorType; + actor.Reset( + NYql::CreateComputeActor( + Options, + Options.MkqlTotalMemoryLimit ? AllocateMemoryFn : nullptr, + Options.MkqlTotalMemoryLimit ? FreeMemoryFn : nullptr, + resultId, + traceId, + std::move(tasks[i]), + computeActorType, + Options.TaskRunnerActorFactory)); + } else { actor.Reset(CreateWorkerActor( - Options.RuntimeData, + Options.RuntimeData, traceId, - Options.TaskRunnerActorFactory, - Options.SourceActorFactory, - Options.SinkActorFactory)); - } - allocationInfo.WorkerActors.emplace_back(RegisterChild( + Options.TaskRunnerActorFactory, + Options.SourceActorFactory, + Options.SinkActorFactory)); + } + allocationInfo.WorkerActors.emplace_back(RegisterChild( actor.Release(), createComputeActor ? NYql::NDq::TEvDq::TEvAbortExecution::Unavailable("Aborted by LWM").Release() : nullptr - )); - } + )); + } - Options.Counters.ActiveWorkers->Add(count); + Options.Counters.ActiveWorkers->Add(count); } - Send(ev->Sender, - MakeHolder<TEvAllocateWorkersResponse>(resourceId, allocationInfo.WorkerActors), - IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, - ev->Cookie); - Subscribe(ev->Sender.NodeId()); + Send(ev->Sender, + MakeHolder<TEvAllocateWorkersResponse>(resourceId, allocationInfo.WorkerActors), + IEventHandle::FlagTrackDelivery | IEventHandle::FlagSubscribeOnSession, + ev->Cookie); + Subscribe(ev->Sender.NodeId()); } void OnFreeWorkers(TEvFreeWorkersNotify::TPtr& ev) { - ui64 resourceId = ev->Get()->Record.GetResourceId(); - YQL_LOG(DEBUG) << "TEvFreeWorkersNotify " << resourceId; - FreeGroup(resourceId, ev->Sender); + ui64 resourceId = ev->Get()->Record.GetResourceId(); + YQL_LOG(DEBUG) << "TEvFreeWorkersNotify " << resourceId; + FreeGroup(resourceId, ev->Sender); } void OnQueryStatus(TEvQueryStatus::TPtr& ev) { - auto response = MakeHolder<TEvQueryStatusResponse>(); - Send(ev->Sender, response.Release()); - } - + auto response = MakeHolder<TEvQueryStatusResponse>(); + Send(ev->Sender, response.Release()); + } + void FreeGroup(ui64 id, NActors::TActorId sender = NActors::TActorId()) { - YQL_LOG(DEBUG) << "Free Group " << id; + YQL_LOG(DEBUG) << "Free Group " << id; auto it = AllocatedWorkers.find(id); if (it != AllocatedWorkers.end()) { for (const auto& actorId : it->second.WorkerActors) { @@ -301,49 +301,49 @@ private: if (sender) { Y_VERIFY(it->second.Sender == sender); } - + MemoryQuoter->Free(it->second.TxId, 0); - Options.Counters.ActiveWorkers->Sub(it->second.WorkerActors.size()); + Options.Counters.ActiveWorkers->Sub(it->second.WorkerActors.size()); AllocatedWorkers.erase(it); } } - - void FreeOnDeadline() { - auto now = TInstant::Now(); - THashSet<ui32> todelete; - for (const auto& [id, info] : AllocatedWorkers) { - if (info.Deadline && info.Deadline < now) { - todelete.insert(id); - } - } - for (const auto& id : todelete) { - YQL_LOG(DEBUG) << "Free on deadline: " << id; - FreeGroup(id); - } - } - - TLocalWorkerManagerOptions Options; - - struct TAllocationInfo { - TVector<NActors::TActorId> WorkerActors; + + void FreeOnDeadline() { + auto now = TInstant::Now(); + THashSet<ui32> todelete; + for (const auto& [id, info] : AllocatedWorkers) { + if (info.Deadline && info.Deadline < now) { + todelete.insert(id); + } + } + for (const auto& id : todelete) { + YQL_LOG(DEBUG) << "Free on deadline: " << id; + FreeGroup(id); + } + } + + TLocalWorkerManagerOptions Options; + + struct TAllocationInfo { + TVector<NActors::TActorId> WorkerActors; NActors::TActorId Sender; - TInstant Deadline; + TInstant Deadline; NDq::TTxId TxId; - }; - THashMap<ui64, TAllocationInfo> AllocatedWorkers; - TDqLocalResourceId ResourceId; - - TRusage Rusage; - + }; + THashMap<ui64, TAllocationInfo> AllocatedWorkers; + TDqLocalResourceId ResourceId; + + TRusage Rusage; + NDq::TAllocateMemoryCallback AllocateMemoryFn; NDq::TFreeMemoryCallback FreeMemoryFn; std::shared_ptr<NDq::TResourceQuoter> MemoryQuoter; -}; - - -NActors::IActor* CreateLocalWorkerManager(const TLocalWorkerManagerOptions& options) -{ - return new TLocalWorkerManager(options); -} - -} // namespace NYql::NDqs +}; + + +NActors::IActor* CreateLocalWorkerManager(const TLocalWorkerManagerOptions& options) +{ + return new TLocalWorkerManager(options); +} + +} // namespace NYql::NDqs diff --git a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h index 5ae92d6b10..144217305f 100644 --- a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h +++ b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.h @@ -8,34 +8,34 @@ #include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h> #include <ydb/library/yql/providers/dq/task_runner/task_runner_invoker.h> - + #include <ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h> - -namespace NYql { - -struct TWorkerRuntimeData; - -} - + +namespace NYql { + +struct TWorkerRuntimeData; + +} + namespace NYql::NDqs { - - struct TLocalWorkerManagerOptions { - TWorkerManagerCounters Counters; - NTaskRunnerProxy::IProxyFactory::TPtr Factory; - NDq::IDqSourceActorFactory::TPtr SourceActorFactory; - NDq::IDqSinkActorFactory::TPtr SinkActorFactory; - TWorkerRuntimeData* RuntimeData = nullptr; - TTaskRunnerInvokerFactory::TPtr TaskRunnerInvokerFactory; - NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; - THashMap<TString, TString> ClusterNamesMapping; - - ui64 MkqlInitialMemoryLimit = 8_GB; - ui64 MkqlTotalMemoryLimit = 0; - ui64 MkqlMinAllocSize = 30_MB; - - bool CanUseComputeActor = true; - }; - - NActors::IActor* CreateLocalWorkerManager(const TLocalWorkerManagerOptions& options); - -} // namespace NYql + + struct TLocalWorkerManagerOptions { + TWorkerManagerCounters Counters; + NTaskRunnerProxy::IProxyFactory::TPtr Factory; + NDq::IDqSourceActorFactory::TPtr SourceActorFactory; + NDq::IDqSinkActorFactory::TPtr SinkActorFactory; + TWorkerRuntimeData* RuntimeData = nullptr; + TTaskRunnerInvokerFactory::TPtr TaskRunnerInvokerFactory; + NDq::NTaskRunnerActor::ITaskRunnerActorFactory::TPtr TaskRunnerActorFactory; + THashMap<TString, TString> ClusterNamesMapping; + + ui64 MkqlInitialMemoryLimit = 8_GB; + ui64 MkqlTotalMemoryLimit = 0; + ui64 MkqlMinAllocSize = 30_MB; + + bool CanUseComputeActor = true; + }; + + NActors::IActor* CreateLocalWorkerManager(const TLocalWorkerManagerOptions& options); + +} // namespace NYql diff --git a/ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h b/ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h index e7ec7ecefc..6c04adec72 100644 --- a/ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h +++ b/ydb/library/yql/providers/dq/worker_manager/worker_manager_common.h @@ -1,44 +1,44 @@ -#pragma once - +#pragma once + #include <ydb/library/yql/providers/dq/worker_manager/interface/events.h> - + #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> - -#include <library/cpp/actors/helpers/future_callback.h> - -namespace NYql::NDqs { - -template<typename TDerived> -class TWorkerManagerCommon: public TRichActor<TDerived> { -public: - TWorkerManagerCommon(void (TDerived::*func)(TAutoPtr<NActors::IEventHandle>& ev, const NActors::TActorContext& ctx)) - : TRichActor<TDerived>(func) - { } - - void OnRoutesRequest(TEvRoutesRequest::TPtr& ev, const NActors::TActorContext& ctx) { - auto localRequest = MakeHolder<NActors::TEvInterconnect::TEvListNodes>(); - - auto replyTo = ev->Sender; - auto* actorSystem = ctx.ExecutorThread.ActorSystem; - - auto callback = MakeHolder<NActors::TActorFutureCallback<NActors::TEvInterconnect::TEvNodesInfo>>( - [replyTo, actorSystem] (TAutoPtr<NActors::TEventHandle<NActors::TEvInterconnect::TEvNodesInfo>>& event) { - auto response = MakeHolder<TEvRoutesResponse>(); - for (const auto& node: event->Get()->Nodes) { - auto* n = response->Record.MutableResponse()->AddNodes(); - n->SetPort(node.Port); - n->SetAddress(node.Address); - n->SetNodeId(node.NodeId); - } - if (actorSystem) { - actorSystem->Send(replyTo, response.Release()); - } - }); - - auto callbackId = ctx.Register(callback.Release()); - - ctx.Send(new NActors::IEventHandle(NActors::GetNameserviceActorId(), callbackId, localRequest.Release())); - } -}; - -} + +#include <library/cpp/actors/helpers/future_callback.h> + +namespace NYql::NDqs { + +template<typename TDerived> +class TWorkerManagerCommon: public TRichActor<TDerived> { +public: + TWorkerManagerCommon(void (TDerived::*func)(TAutoPtr<NActors::IEventHandle>& ev, const NActors::TActorContext& ctx)) + : TRichActor<TDerived>(func) + { } + + void OnRoutesRequest(TEvRoutesRequest::TPtr& ev, const NActors::TActorContext& ctx) { + auto localRequest = MakeHolder<NActors::TEvInterconnect::TEvListNodes>(); + + auto replyTo = ev->Sender; + auto* actorSystem = ctx.ExecutorThread.ActorSystem; + + auto callback = MakeHolder<NActors::TActorFutureCallback<NActors::TEvInterconnect::TEvNodesInfo>>( + [replyTo, actorSystem] (TAutoPtr<NActors::TEventHandle<NActors::TEvInterconnect::TEvNodesInfo>>& event) { + auto response = MakeHolder<TEvRoutesResponse>(); + for (const auto& node: event->Get()->Nodes) { + auto* n = response->Record.MutableResponse()->AddNodes(); + n->SetPort(node.Port); + n->SetAddress(node.Address); + n->SetNodeId(node.NodeId); + } + if (actorSystem) { + actorSystem->Send(replyTo, response.Release()); + } + }); + + auto callbackId = ctx.Register(callback.Release()); + + ctx.Send(new NActors::IEventHandle(NActors::GetNameserviceActorId(), callbackId, localRequest.Release())); + } +}; + +} diff --git a/ydb/library/yql/providers/dq/worker_manager/ya.make b/ydb/library/yql/providers/dq/worker_manager/ya.make index c28fcecc7b..fffe801bb0 100644 --- a/ydb/library/yql/providers/dq/worker_manager/ya.make +++ b/ydb/library/yql/providers/dq/worker_manager/ya.make @@ -1,8 +1,8 @@ -LIBRARY() - +LIBRARY() + OWNER(g:yql) - -PEERDIR( + +PEERDIR( ydb/core/kqp ydb/library/yql/utils/failure_injector ydb/library/yql/providers/common/config @@ -14,12 +14,12 @@ PEERDIR( ydb/library/yql/providers/dq/task_runner ydb/library/yql/providers/dq/task_runner_actor ydb/library/yql/providers/dq/worker_manager/interface -) - +) + YQL_LAST_ABI_VERSION() - -SRCS( - local_worker_manager.cpp -) - -END() + +SRCS( + local_worker_manager.cpp +) + +END() diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make index 743cb1e250..986edc777d 100644 --- a/ydb/library/yql/providers/dq/ya.make +++ b/ydb/library/yql/providers/dq/ya.make @@ -1,19 +1,19 @@ -RECURSE( - actors - api +RECURSE( + actors + api backtrace - common + common config counters - expr_nodes + expr_nodes interface mkql - opt - planner - provider + opt + planner + provider provider/exec runtime task_runner - task_runner_actor - worker_manager -) + task_runner_actor + worker_manager +) diff --git a/ydb/library/yql/providers/pq/provider/ut/ya.make b/ydb/library/yql/providers/pq/provider/ut/ya.make index b1e1736c00..66015a4f05 100644 --- a/ydb/library/yql/providers/pq/provider/ut/ya.make +++ b/ydb/library/yql/providers/pq/provider/ut/ya.make @@ -18,7 +18,7 @@ PEERDIR( ydb/library/yql/dq/comp_nodes ydb/library/yql/providers/common/comp_nodes ydb/library/yql/providers/dq/provider - ydb/library/yql/providers/dq/local_gateway + ydb/library/yql/providers/dq/local_gateway ydb/library/yql/providers/pq/gateway/dummy ydb/library/yql/providers/pq/provider ydb/library/yql/providers/solomon/gateway diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp index 21f11e03fd..ea67ffd422 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp @@ -65,7 +65,7 @@ public: return PartitionTopicRead(topicSource.Cast().Topic(), maxPartitions, partitions); } } - return 0; + return 0; } TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) override { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp index 1e0e6e6a61..4127452d12 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp @@ -23,7 +23,7 @@ #include <ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h> -#include <ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h> +#include <ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.h> #include <ydb/library/yql/core/facade/yql_facade.h> #include <ydb/library/yql/utils/log/log.h> diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp index 65bf8fb12c..13bfd5e0f7 100644 --- a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp +++ b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp @@ -259,11 +259,11 @@ namespace { output = input; auto status = DelegatedProvider->GetCallableExecutionTransformer() .ApplyAsyncChanges(DelegatedNode, DelegatedNodeOutput, ctx); - if (status == TStatus::Repeat && input != DelegatedNodeOutput->TailPtr()) { - output = DelegatedNodeOutput->TailPtr(); - } else { - FinishNode(*input, ctx, status); - } + if (status == TStatus::Repeat && input != DelegatedNodeOutput->TailPtr()) { + output = DelegatedNodeOutput->TailPtr(); + } else { + FinishNode(*input, ctx, status); + } return status; } @@ -1379,14 +1379,14 @@ namespace { return *CallableExecutionTransformer; } - void Reset() final { - TDataProviderBase::Reset(); - if (CallableExecutionTransformer) { - CallableExecutionTransformer.Reset(); - } - Config->CommittedResults.clear(); - } - + void Reset() final { + TDataProviderBase::Reset(); + if (CallableExecutionTransformer) { + CallableExecutionTransformer.Reset(); + } + Config->CommittedResults.clear(); + } + bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override { if (CanExecute(node)) { children.push_back(node.ChildPtr(0)); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp index c574e8f880..ecb85e824b 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp @@ -66,7 +66,7 @@ public: TStringOutput out(partitions.back()); range.Save(&out); } - return 0; + return 0; } TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode& read, TExprContext&, bool ) override { diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp index 1ce290133a..607b3ba7a8 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_load_meta.cpp @@ -216,7 +216,7 @@ public: snapshots.emplace(client.first, snapshot.ExtractResult()); } else { failed = true; - const auto& config = State_->Configuration->Clusters[TString(client.first)]; + const auto& config = State_->Configuration->Clusters[TString(client.first)]; ctx.AddError(TIssue({}, TStringBuilder() << "Failed to take snapshot for: `" << client.first << "`, endpoint: " << config.Endpoint << ", status: " << snapshot.GetStatus())); for (const auto& issue : snapshot.GetIssues()) ctx.AddError(issue); diff --git a/ydb/library/yql/sql/settings/translation_settings.cpp b/ydb/library/yql/sql/settings/translation_settings.cpp index f60d8b40da..e09fd43aaa 100644 --- a/ydb/library/yql/sql/settings/translation_settings.cpp +++ b/ydb/library/yql/sql/settings/translation_settings.cpp @@ -14,9 +14,9 @@ namespace { } using namespace NSQLTranslation; - class TAlwaysDiallowPolicy : public ISqlFeaturePolicy { + class TAlwaysDiallowPolicy : public ISqlFeaturePolicy { public: - TAlwaysDiallowPolicy() = default; + TAlwaysDiallowPolicy() = default; bool Allow() const override { return false; } @@ -24,8 +24,8 @@ namespace { } namespace NSQLTranslation { - ISqlFeaturePolicy::TPtr ISqlFeaturePolicy::MakeAlwaysDisallow() { - return new TAlwaysDiallowPolicy; + ISqlFeaturePolicy::TPtr ISqlFeaturePolicy::MakeAlwaysDisallow() { + return new TAlwaysDiallowPolicy; } TTranslationSettings::TTranslationSettings() @@ -41,8 +41,8 @@ namespace NSQLTranslation { , V0Behavior(EV0Behavior::Silent) , V0ForceDisable(InTestEnvironment()) , WarnOnV0(true) - , V0WarnAsError(ISqlFeaturePolicy::MakeAlwaysDisallow()) - , DqDefaultAuto(ISqlFeaturePolicy::MakeAlwaysDisallow()) + , V0WarnAsError(ISqlFeaturePolicy::MakeAlwaysDisallow()) + , DqDefaultAuto(ISqlFeaturePolicy::MakeAlwaysDisallow()) , AssumeYdbOnClusterWithSlash(false) {} diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h index 1f53e14643..0b9194a249 100644 --- a/ydb/library/yql/sql/settings/translation_settings.h +++ b/ydb/library/yql/sql/settings/translation_settings.h @@ -34,14 +34,14 @@ namespace NSQLTranslation { Disable }; - class ISqlFeaturePolicy : public TThrRefBase { + class ISqlFeaturePolicy : public TThrRefBase { public: - virtual ~ISqlFeaturePolicy() = default; - virtual bool Allow() const = 0; + virtual ~ISqlFeaturePolicy() = default; + virtual bool Allow() const = 0; - using TPtr = TIntrusivePtr<ISqlFeaturePolicy>; + using TPtr = TIntrusivePtr<ISqlFeaturePolicy>; - static TPtr MakeAlwaysDisallow(); + static TPtr MakeAlwaysDisallow(); }; struct TTableBindingSettings { @@ -79,8 +79,8 @@ namespace NSQLTranslation { EV0Behavior V0Behavior; bool V0ForceDisable; bool WarnOnV0; - ISqlFeaturePolicy::TPtr V0WarnAsError; - ISqlFeaturePolicy::TPtr DqDefaultAuto; + ISqlFeaturePolicy::TPtr V0WarnAsError; + ISqlFeaturePolicy::TPtr DqDefaultAuto; bool AssumeYdbOnClusterWithSlash; }; diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp index 92c3f4288f..b028fc0439 100644 --- a/ydb/library/yql/sql/v0/sql.cpp +++ b/ydb/library/yql/sql/v0/sql.cpp @@ -5052,7 +5052,7 @@ TNodePtr TSqlQuery::Build(const TSQLParserAST& ast) { TVector<TNodePtr> blocks; if (Ctx.Settings.WarnOnV0) { - if (Ctx.Settings.V0WarnAsError->Allow()) { + if (Ctx.Settings.V0WarnAsError->Allow()) { Error() << "SQL v0 syntax is deprecated and no longer supported. Please switch to v1: https://clubs.at.yandex-team.ru/yql/2910"; return nullptr; } diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp index f8e9c5a49b..c08fe96cae 100644 --- a/ydb/library/yql/sql/v1/context.cpp +++ b/ydb/library/yql/sql/v1/context.cpp @@ -45,7 +45,7 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = { {"AnsiOptionalAs", &TContext::AnsiOptionalAs}, {"WarnOnAnsiAliasShadowing", &TContext::WarnOnAnsiAliasShadowing}, {"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin}, - {"DqEngineEnable", &TContext::DqEngineEnable}, + {"DqEngineEnable", &TContext::DqEngineEnable}, {"DqEngineForce", &TContext::DqEngineForce}, {"RegexUseRe2", &TContext::PragmaRegexUseRe2}, {"OrderedColumns", &TContext::OrderedColumns}, @@ -75,7 +75,7 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings, , Issues(issues) , IncrementMonCounterFunction(settings.IncrementCounter) , HasPendingErrors(false) - , DqEngineEnable(Settings.DqDefaultAuto->Allow()) + , DqEngineEnable(Settings.DqDefaultAuto->Allow()) , AnsiQuotedIdentifiers(settings.AnsiLexer) { for (auto lib : settings.Libraries) { diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h index 7f2142da01..8951757486 100644 --- a/ydb/library/yql/sql/v1/context.h +++ b/ydb/library/yql/sql/v1/context.h @@ -234,8 +234,8 @@ namespace NSQLTranslationV1 { bool WarnUnnamedColumns = false; bool DiscoveryMode = false; bool EnableSystemColumns = true; - bool DqEngineEnable = false; - bool DqEngineForce = false; + bool DqEngineEnable = false; + bool DqEngineForce = false; TMaybe<bool> JsonQueryReturnsJsonDocument; TMaybe<bool> AnsiInForEmptyOrNullableItemsCollections; TMaybe<bool> AnsiRankForNullableKeys = true; diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp index cf6244c0e9..725356b9f5 100644 --- a/ydb/library/yql/sql/v1/query.cpp +++ b/ydb/library/yql/sql/v1/query.cpp @@ -1660,7 +1660,7 @@ public: Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "DiscoveryMode")))); } - + if (ctx.DqEngineEnable) { TString mode = "auto"; if (ctx.PqReadByRtmrCluster && ctx.PqReadByRtmrCluster != "dq") { @@ -1668,10 +1668,10 @@ public: } else if (ctx.DqEngineForce) { mode = "force"; } - Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, + Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode)))); - } - + } + if (ctx.JsonQueryReturnsJsonDocument.Defined()) { TString pragmaName = "DisableJsonQueryReturnsJsonDocument"; if (*ctx.JsonQueryReturnsJsonDocument) { diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp index fd64ca9c9b..ab5a864788 100644 --- a/ydb/library/yql/sql/v1/sql.cpp +++ b/ydb/library/yql/sql/v1/sql.cpp @@ -9473,25 +9473,25 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success } else if (normalizedPragma == "disableansiinforemptyornullableitemscollections") { Ctx.AnsiInForEmptyOrNullableItemsCollections = false; Ctx.IncrementMonCounter("sql_pragma", "DisableAnsiInForEmptyOrNullableItemsCollections"); - } else if (normalizedPragma == "dqengine") { - Ctx.IncrementMonCounter("sql_pragma", "DqEngine"); + } else if (normalizedPragma == "dqengine") { + Ctx.IncrementMonCounter("sql_pragma", "DqEngine"); if (values.size() != 1 || !values[0].GetLiteral() - || ! (*values[0].GetLiteral() == "disable" || *values[0].GetLiteral() == "auto" || *values[0].GetLiteral() == "force")) - { - Error() << "Expected `disable|auto|force' argument for: " << pragma; + || ! (*values[0].GetLiteral() == "disable" || *values[0].GetLiteral() == "auto" || *values[0].GetLiteral() == "force")) + { + Error() << "Expected `disable|auto|force' argument for: " << pragma; Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue"); return {}; - } - if (*values[0].GetLiteral() == "disable") { - Ctx.DqEngineEnable = false; - Ctx.DqEngineForce = false; - } else if (*values[0].GetLiteral() == "force") { - Ctx.DqEngineEnable = true; - Ctx.DqEngineForce = true; - } else if (*values[0].GetLiteral() == "auto") { - Ctx.DqEngineEnable = true; - Ctx.DqEngineForce = false; - } + } + if (*values[0].GetLiteral() == "disable") { + Ctx.DqEngineEnable = false; + Ctx.DqEngineForce = false; + } else if (*values[0].GetLiteral() == "force") { + Ctx.DqEngineEnable = true; + Ctx.DqEngineForce = true; + } else if (*values[0].GetLiteral() == "auto") { + Ctx.DqEngineEnable = true; + Ctx.DqEngineForce = false; + } } else if (normalizedPragma == "ansirankfornullablekeys") { Ctx.AnsiRankForNullableKeys = true; Ctx.IncrementMonCounter("sql_pragma", "AnsiRankForNullableKeys"); diff --git a/ydb/library/yql/utils/actor_log/log.cpp b/ydb/library/yql/utils/actor_log/log.cpp index f7738acd0f..3b3002208c 100644 --- a/ydb/library/yql/utils/actor_log/log.cpp +++ b/ydb/library/yql/utils/actor_log/log.cpp @@ -1,15 +1,15 @@ -#include "log.h" +#include "log.h" #include <library/cpp/actors/core/log.h> -namespace NYql { -namespace NDq { +namespace NYql { +namespace NDq { + +using namespace NActors; -using namespace NActors; - namespace { -NActors::NLog::EPriority GetActorLogPriority(ELogPriority priority) { +NActors::NLog::EPriority GetActorLogPriority(ELogPriority priority) { switch (priority) { case TLOG_EMERG: return NActors::NLog::PRI_EMERG; @@ -56,7 +56,7 @@ NYql::NLog::ELevel GetYqlLogLevel(NActors::NLog::EPriority priority) { } // namespace -void TActorYqlLogBackend::WriteData(const TLogRecord& rec) { +void TActorYqlLogBackend::WriteData(const TLogRecord& rec) { std::visit([&](const auto* actorCtxOrSystem){ Y_VERIFY(actorCtxOrSystem); if (TraceId.empty()) { @@ -77,5 +77,5 @@ void SetYqlLogLevels(const NActors::NLog::EPriority& priority) { }); } -} // namespace NDq -} // namespace NYql +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/utils/actor_log/log.h b/ydb/library/yql/utils/actor_log/log.h index 4bd6639c5e..08d525d28b 100644 --- a/ydb/library/yql/utils/actor_log/log.h +++ b/ydb/library/yql/utils/actor_log/log.h @@ -1,26 +1,26 @@ -#pragma once +#pragma once #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/log/tls_backend.h> - + #include <ydb/core/protos/services.pb.h> - -#include <library/cpp/actors/core/actorsystem.h> - -namespace NYql { -namespace NDq { - -class TActorYqlLogBackend : public TLogBackend { -public: + +#include <library/cpp/actors/core/actorsystem.h> + +namespace NYql { +namespace NDq { + +class TActorYqlLogBackend : public TLogBackend { +public: TActorYqlLogBackend( const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId) : ActorCtxOrSystem(&actorCtx) - , Component(component) - , SessionId(sessionId) - , TraceId(traceId) {} - + , Component(component) + , SessionId(sessionId) + , TraceId(traceId) {} + TActorYqlLogBackend( const NActors::TActorSystem* actorSystem, int component, @@ -31,49 +31,49 @@ public: , SessionId(sessionId) , TraceId(traceId) {} - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override {} - -private: + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override {} + +private: std::variant<const NActors::TActorContext*, const NActors::TActorSystem*> ActorCtxOrSystem; - int Component; - TString SessionId; - TString TraceId; -}; - -class TYqlLogScope : public NYql::NLog::TScopedBackend<TActorYqlLogBackend> { -public: + int Component; + TString SessionId; + TString TraceId; +}; + +class TYqlLogScope : public NYql::NLog::TScopedBackend<TActorYqlLogBackend> { +public: TYqlLogScope(const NActors::TActorContext& actorCtx, int component, const TString& sessionId, const TString& traceId = "") - : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorCtx, component, sessionId, traceId) {}; + : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorCtx, component, sessionId, traceId) {}; TYqlLogScope(const NActors::TActorSystem* actorSystem, int component, const TString& sessionId, const TString& traceId = "") : NYql::NLog::TScopedBackend<TActorYqlLogBackend>(actorSystem, component, sessionId, traceId) {}; -}; - -class TLogWrapReceive: public NActors::TDecorator { -public: - TLogWrapReceive(NActors::IActor* actor, const TString& sessionId, int component = NKikimrServices::YQL_PROXY) +}; + +class TLogWrapReceive: public NActors::TDecorator { +public: + TLogWrapReceive(NActors::IActor* actor, const TString& sessionId, int component = NKikimrServices::YQL_PROXY) : NActors::TDecorator(THolder(actor)) - , SessionId(sessionId) - , Component(component) - { } - - bool DoBeforeReceiving(TAutoPtr<NActors::IEventHandle>& /*ev*/, const NActors::TActorContext& ctx) override { - LogScope.ConstructInPlace(ctx, Component, SessionId); - return true; - } - - void DoAfterReceiving(const NActors::TActorContext& /*ctx*/) override { - LogScope.Clear(); - } - -private: - TString SessionId; - int Component; - TMaybe<TYqlLogScope> LogScope; -}; - -void SetYqlLogLevels(const NActors::NLog::EPriority& priority); - -} // namespace NDq -} // namespace NYql + , SessionId(sessionId) + , Component(component) + { } + + bool DoBeforeReceiving(TAutoPtr<NActors::IEventHandle>& /*ev*/, const NActors::TActorContext& ctx) override { + LogScope.ConstructInPlace(ctx, Component, SessionId); + return true; + } + + void DoAfterReceiving(const NActors::TActorContext& /*ctx*/) override { + LogScope.Clear(); + } + +private: + TString SessionId; + int Component; + TMaybe<TYqlLogScope> LogScope; +}; + +void SetYqlLogLevels(const NActors::NLog::EPriority& priority); + +} // namespace NDq +} // namespace NYql diff --git a/ydb/library/yql/utils/actor_log/ya.make b/ydb/library/yql/utils/actor_log/ya.make index e6177608a8..cae3dcc827 100644 --- a/ydb/library/yql/utils/actor_log/ya.make +++ b/ydb/library/yql/utils/actor_log/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - -OWNER(g:yql) - +LIBRARY() + +OWNER(g:yql) + SRCS( log.cpp ) - -PEERDIR( + +PEERDIR( library/cpp/actors/core - library/cpp/actors/protos + library/cpp/actors/protos ydb/core/protos -) - -END() +) + +END() diff --git a/ydb/public/api/grpc/draft/ya.make b/ydb/public/api/grpc/draft/ya.make index a669898ca1..f63be96521 100644 --- a/ydb/public/api/grpc/draft/ya.make +++ b/ydb/public/api/grpc/draft/ya.make @@ -20,7 +20,7 @@ SRCS( ydb_s3_internal_v1.proto ydb_long_tx_v1.proto ydb_logstore_v1.proto - yql_db_v1.proto + yql_db_v1.proto ) PEERDIR( diff --git a/ydb/public/api/grpc/draft/yql_db_v1.proto b/ydb/public/api/grpc/draft/yql_db_v1.proto index dbe223bacb..12b061eca3 100644 --- a/ydb/public/api/grpc/draft/yql_db_v1.proto +++ b/ydb/public/api/grpc/draft/yql_db_v1.proto @@ -1,19 +1,19 @@ -syntax = "proto3"; - +syntax = "proto3"; + package Yq.Private.V1; - + import "ydb/public/api/protos/draft/yq_private.proto"; - + service YqPrivateTaskService { - // gets new task + // gets new task rpc GetTask(Yq.Private.GetTaskRequest) returns (Yq.Private.GetTaskResponse); - - // pings new task (also can update metadata) + + // pings new task (also can update metadata) rpc PingTask(Yq.Private.PingTaskRequest) returns (Yq.Private.PingTaskResponse); - - // writes rows + + // writes rows rpc WriteTaskResult(Yq.Private.WriteTaskResultRequest) returns (Yq.Private.WriteTaskResultResponse); //Nodes rpc NodesHealthCheck(Yq.Private.NodesHealthCheckRequest) returns (Yq.Private.NodesHealthCheckResponse); -} +} diff --git a/ydb/public/api/protos/draft/yq_private.proto b/ydb/public/api/protos/draft/yq_private.proto index 5612f1f728..2af1c2f3ed 100644 --- a/ydb/public/api/protos/draft/yq_private.proto +++ b/ydb/public/api/protos/draft/yq_private.proto @@ -1,25 +1,25 @@ -syntax = "proto3"; -option cc_enable_arenas = true; - +syntax = "proto3"; +option cc_enable_arenas = true; + package Yq.Private; - + import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_value.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; - + import "ydb/public/api/protos/yq.proto"; - -import "google/protobuf/timestamp.proto"; - -//////////////////////////////////////////////////////////// - -message GetTaskRequest { + +import "google/protobuf/timestamp.proto"; + +//////////////////////////////////////////////////////////// + +message GetTaskRequest { string tenant = 1; string owner_id = 2; // guid, should be refreshed on node restart string host = 3; Ydb.Operations.OperationParams operation_params = 4; -} - +} + message SignedIdentity { string value = 1; string signature = 2; @@ -36,21 +36,21 @@ message TopicConsumer { bool add_bearer_to_token = 8; } -message GetTaskResult { +message GetTaskResult { message Task { // come back later in 10 sec ? SignedIdentity result_id = 1; SignedIdentity query_id = 2; SignedIdentity job_id = 3; uint64 generation = 4; - + bool streaming = 5; repeated bytes dq_graph = 6; // text, connection and binding are empty if dq_graph is not empty string text = 7; repeated YandexQuery.Connection connection = 8; repeated YandexQuery.Binding binding = 9; - + string user_token = 10; // IAM token for debug repeated SignedIdentity service_accounts = 11; string user_id = 12; @@ -70,24 +70,24 @@ message GetTaskResult { YandexQuery.StreamingDisposition disposition = 25; } repeated Task tasks = 1; -} - -message GetTaskResponse { - Ydb.Operations.Operation operation = 1; // GetTaskResult -} - -message PingTaskRequest { +} + +message GetTaskResponse { + Ydb.Operations.Operation operation = 1; // GetTaskResult +} + +message PingTaskRequest { string owner_id = 1; SignedIdentity query_id = 2; SignedIdentity job_id = 3; SignedIdentity result_id = 4; YandexQuery.QueryMeta.ComputeStatus status = 5; - repeated Ydb.Issue.IssueMessage issues = 6; + repeated Ydb.Issue.IssueMessage issues = 6; repeated Ydb.Issue.IssueMessage transient_issues = 16; - uint32 result_set_count = 7; - string statistics = 8; + uint32 result_set_count = 7; + string statistics = 8; repeated YandexQuery.ResultSetMeta result_set_meta = 9; - string executer_info = 10; + string executer_info = 10; repeated bytes dq_graph = 11; int32 dq_graph_index = 20; string ast = 12; @@ -101,34 +101,34 @@ message PingTaskRequest { google.protobuf.Timestamp started_at = 101; google.protobuf.Timestamp finished_at = 102; google.protobuf.Timestamp deadline = 103; -} - -message PingTaskResult { +} + +message PingTaskResult { YandexQuery.QueryAction action = 1; -} - -message PingTaskResponse { - Ydb.Operations.Operation operation = 1; // PingTaskResult -} - -message WriteTaskResultRequest { +} + +message PingTaskResponse { + Ydb.Operations.Operation operation = 1; // PingTaskResult +} + +message WriteTaskResultRequest { string owner_id = 1; SignedIdentity result_id = 2; - Ydb.ResultSet result_set = 3; - uint32 result_set_id = 4; - uint64 offset = 5; + Ydb.ResultSet result_set = 3; + uint32 result_set_id = 4; + uint64 offset = 5; uint64 request_id = 6; Ydb.Operations.OperationParams operation_params = 7; google.protobuf.Timestamp deadline = 8; -} - -message WriteTaskResultResult { +} + +message WriteTaskResultResult { uint64 request_id = 1; -} - -message WriteTaskResultResponse { - Ydb.Operations.Operation operation = 1; // WriteRowsResultResult -} +} + +message WriteTaskResultResponse { + Ydb.Operations.Operation operation = 1; // WriteRowsResultResult +} message NodeInfo { uint32 node_id = 1; diff --git a/ydb/public/api/protos/ydb_value.proto b/ydb/public/api/protos/ydb_value.proto index d590515e40..d885d73f8e 100644 --- a/ydb/public/api/protos/ydb_value.proto +++ b/ydb/public/api/protos/ydb_value.proto @@ -45,11 +45,11 @@ message DictType { Type payload = 2; } -message TaggedType { - string tag = 1; - Type type = 2; -} - +message TaggedType { + string tag = 1; + Type type = 2; +} + message Type { enum PrimitiveTypeId { PRIMITIVE_TYPE_ID_UNSPECIFIED = 0x0000; @@ -92,13 +92,13 @@ message Type { StructType struct_type = 104; DictType dict_type = 105; VariantType variant_type = 106; - TaggedType tagged_type = 107; + TaggedType tagged_type = 107; /* Special types */ google.protobuf.NullValue void_type = 201; - google.protobuf.NullValue null_type = 202; - google.protobuf.NullValue empty_list_type = 203; - google.protobuf.NullValue empty_dict_type = 204; + google.protobuf.NullValue null_type = 202; + google.protobuf.NullValue empty_list_type = 203; + google.protobuf.NullValue empty_dict_type = 204; } } diff --git a/ydb/public/lib/yq/helpers.h b/ydb/public/lib/yq/helpers.h index 3f30ab5831..f07331f38f 100644 --- a/ydb/public/lib/yq/helpers.h +++ b/ydb/public/lib/yq/helpers.h @@ -1,25 +1,25 @@ -#pragma once - -#include "scope.h" - -namespace NYdb { -namespace NYq { - -template<typename T> +#pragma once + +#include "scope.h" + +namespace NYdb { +namespace NYq { + +template<typename T> T CreateYqSettings(const TString& folderId) -{ - T settings; +{ + T settings; settings.Header_ = {{ "x-yq-scope", TScope::YandexCloudScopeSchema + "://" + folderId }}; - return settings; -} - -template<typename T> -T CreateYqSettings(const TScope& scope) -{ - T settings; - settings.Header_ = {{ "x-yq-scope", scope.ToString() }}; - return settings; -} - -} // namespace NYq -} // namespace Ndb + return settings; +} + +template<typename T> +T CreateYqSettings(const TScope& scope) +{ + T settings; + settings.Header_ = {{ "x-yq-scope", scope.ToString() }}; + return settings; +} + +} // namespace NYq +} // namespace Ndb diff --git a/ydb/public/lib/yq/scope.cpp b/ydb/public/lib/yq/scope.cpp index 74d64e059d..7cd192eec1 100644 --- a/ydb/public/lib/yq/scope.cpp +++ b/ydb/public/lib/yq/scope.cpp @@ -1,9 +1,9 @@ -#include "scope.h" - -namespace NYdb { -namespace NYq { - -TString TScope::YandexCloudScopeSchema = "yandexcloud"; - -} // namespace NYq -} // namespace Ndb +#include "scope.h" + +namespace NYdb { +namespace NYq { + +TString TScope::YandexCloudScopeSchema = "yandexcloud"; + +} // namespace NYq +} // namespace Ndb diff --git a/ydb/public/lib/yq/scope.h b/ydb/public/lib/yq/scope.h index c0774998aa..4afdcdb404 100644 --- a/ydb/public/lib/yq/scope.h +++ b/ydb/public/lib/yq/scope.h @@ -1,36 +1,36 @@ -#pragma once - -#include <util/string/builder.h> - -namespace NYdb { -namespace NYq { - -class TScope { -public: - static TString YandexCloudScopeSchema; - - TScope() - { } - - TScope(const TString& scope) - : Scope(scope) - { } - - TScope(TString&& scope) - : Scope(std::move(scope)) - { } - - const TString& ToString() const { - return Scope; - } - - bool Empty() const { - return Scope.empty(); - } - -private: - TString Scope; -}; - -} // namespace NYq -} // namespace Ndb +#pragma once + +#include <util/string/builder.h> + +namespace NYdb { +namespace NYq { + +class TScope { +public: + static TString YandexCloudScopeSchema; + + TScope() + { } + + TScope(const TString& scope) + : Scope(scope) + { } + + TScope(TString&& scope) + : Scope(std::move(scope)) + { } + + const TString& ToString() const { + return Scope; + } + + bool Empty() const { + return Scope.empty(); + } + +private: + TString Scope; +}; + +} // namespace NYq +} // namespace Ndb diff --git a/ydb/public/lib/yq/ya.make b/ydb/public/lib/yq/ya.make index 07236e96ed..df334eef0c 100644 --- a/ydb/public/lib/yq/ya.make +++ b/ydb/public/lib/yq/ya.make @@ -7,7 +7,7 @@ OWNER( SRCS( yq.cpp - scope.cpp + scope.cpp ) PEERDIR( diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index b63892aea3..8c18d4d50f 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -50,14 +50,14 @@ static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) { return ETypeKind::Variant; case Ydb::Type::kVoidType: return ETypeKind::Void; - case Ydb::Type::kNullType: - return ETypeKind::Null; - case Ydb::Type::kEmptyListType: - return ETypeKind::EmptyList; - case Ydb::Type::kEmptyDictType: - return ETypeKind::EmptyDict; - case Ydb::Type::kTaggedType: - return ETypeKind::Tagged; + case Ydb::Type::kNullType: + return ETypeKind::Null; + case Ydb::Type::kEmptyListType: + return ETypeKind::EmptyList; + case Ydb::Type::kEmptyDictType: + return ETypeKind::EmptyDict; + case Ydb::Type::kTaggedType: + return ETypeKind::Tagged; default: break; } @@ -203,11 +203,11 @@ public: ForwardStep(); } - const TString& GetTag() { - CheckPreviousKind(ETypeKind::Tagged, "GetTag"); - return GetProto(1).tagged_type().tag(); - } - + const TString& GetTag() { + CheckPreviousKind(ETypeKind::Tagged, "GetTag"); + return GetProto(1).tagged_type().tag(); + } + bool ForwardStep() { auto& idx = Path_.back().Idx; const google::protobuf::Message* nextPtr = nullptr; @@ -265,33 +265,33 @@ public: break; } - case ETypeKind::Variant: { - const Ydb::VariantType& variantType = GetProto().variant_type(); - auto wrappedVariant = std::make_unique<Ydb::Type>(); - switch (variantType.type_case()) { - case Ydb::VariantType::kTupleItems: { - *wrappedVariant->mutable_tuple_type() = variantType.tuple_items(); - break; - } - case Ydb::VariantType::kStructItems: { - *wrappedVariant->mutable_struct_type() = variantType.struct_items(); - break; - } - default: { - FatalError(TStringBuilder() << "Unexpected variant type kind: " << variantType); - break; - } - } - WrappedVariants_.emplace_back(std::move(wrappedVariant)); - nextPtr = WrappedVariants_.back().get(); - break; - } - - case ETypeKind::Tagged: { - nextPtr = &GetProto().tagged_type().type(); - break; - } - + case ETypeKind::Variant: { + const Ydb::VariantType& variantType = GetProto().variant_type(); + auto wrappedVariant = std::make_unique<Ydb::Type>(); + switch (variantType.type_case()) { + case Ydb::VariantType::kTupleItems: { + *wrappedVariant->mutable_tuple_type() = variantType.tuple_items(); + break; + } + case Ydb::VariantType::kStructItems: { + *wrappedVariant->mutable_struct_type() = variantType.struct_items(); + break; + } + default: { + FatalError(TStringBuilder() << "Unexpected variant type kind: " << variantType); + break; + } + } + WrappedVariants_.emplace_back(std::move(wrappedVariant)); + nextPtr = WrappedVariants_.back().get(); + break; + } + + case ETypeKind::Tagged: { + nextPtr = &GetProto().tagged_type().type(); + break; + } + default: FatalError(TStringBuilder() << "Unexpected type kind: " << GetKind()); break; @@ -336,7 +336,7 @@ private: private: TType Type_; TStackVec<TProtoPosition, 8> Path_; - TStackVec<std::unique_ptr<Ydb::Type>, 8> WrappedVariants_; + TStackVec<std::unique_ptr<Ydb::Type>, 8> WrappedVariants_; }; //////////////////////////////////////////////////////////////////////////////// @@ -423,26 +423,26 @@ void TTypeParser::OpenVariant(size_t index) { Impl_->OpenVariant(index); } -void TTypeParser::OpenVariant() { - Impl_->Open<ETypeKind::Variant>(); -} - +void TTypeParser::OpenVariant() { + Impl_->Open<ETypeKind::Variant>(); +} + void TTypeParser::CloseVariant() { Impl_->Close<ETypeKind::Variant>(); } -void TTypeParser::OpenTagged() { - Impl_->Open<ETypeKind::Tagged>(); -} - -const TString& TTypeParser::GetTag() { - return Impl_->GetTag(); -} - -void TTypeParser::CloseTagged() { - Impl_->Close<ETypeKind::Tagged>(); -} - +void TTypeParser::OpenTagged() { + Impl_->Open<ETypeKind::Tagged>(); +} + +const TString& TTypeParser::GetTag() { + return Impl_->GetTag(); +} + +void TTypeParser::CloseTagged() { + Impl_->Close<ETypeKind::Tagged>(); +} + //////////////////////////////////////////////////////////////////////////////// void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { @@ -1218,18 +1218,18 @@ public: TypeParser_.CloseVariant(); } - void OpenTagged() { - TypeParser_.OpenTagged(); - } - - const TString& GetTag() { - return TypeParser_.GetTag(); - } - - void CloseTagged() { - TypeParser_.CloseTagged(); - } - + void OpenTagged() { + TypeParser_.OpenTagged(); + } + + const TString& GetTag() { + return TypeParser_.GetTag(); + } + + void CloseTagged() { + TypeParser_.CloseTagged(); + } + private: const TProtoPosition& GetPathBack() const { if (Path_.empty()) { @@ -1749,18 +1749,18 @@ void TValueParser::CloseVariant() { Impl_->CloseVariant(); } -void TValueParser::OpenTagged() { - Impl_->OpenTagged(); -} - -const TString& TValueParser::GetTag() const { - return Impl_->GetTag(); -} - -void TValueParser::CloseTagged() { - Impl_->CloseTagged(); -} - +void TValueParser::OpenTagged() { + Impl_->OpenTagged(); +} + +const TString& TValueParser::GetTag() const { + return Impl_->GetTag(); +} + +void TValueParser::CloseTagged() { + Impl_->CloseTagged(); +} + //////////////////////////////////////////////////////////////////////////////// class TValueBuilderImpl { diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index b2f065f153..4a163b9220 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -82,11 +82,11 @@ public: Struct, Dict, Variant, - Void, - Null, - EmptyList, - EmptyDict, - Tagged + Void, + Null, + EmptyList, + EmptyDict, + Tagged }; public: @@ -127,14 +127,14 @@ public: // Variant void OpenVariant(size_t index); - void OpenVariant(); + void OpenVariant(); void CloseVariant(); - // Tagged - void OpenTagged(); - const TString& GetTag(); - void CloseTagged(); - + // Tagged + void OpenTagged(); + const TString& GetTag(); + void CloseTagged(); + private: class TImpl; std::unique_ptr<TImpl> Impl_; @@ -318,11 +318,11 @@ public: void OpenVariant(); void CloseVariant(); - // Tagged - void OpenTagged(); - const TString& GetTag() const; - void CloseTagged(); - + // Tagged + void OpenTagged(); + const TString& GetTag() const; + void CloseTagged(); + private: TValueParser(const TType& type); void Reset(const Ydb::Value& value); diff --git a/ydb/services/yq/ut_integration/ut_utils.cpp b/ydb/services/yq/ut_integration/ut_utils.cpp index 21a681d941..925f74f7b3 100644 --- a/ydb/services/yq/ut_integration/ut_utils.cpp +++ b/ydb/services/yq/ut_integration/ut_utils.cpp @@ -1,93 +1,93 @@ -#include "ut_utils.h" - +#include "ut_utils.h" + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> #include <ydb/public/lib/yq/scope.h> #include <ydb/core/yq/libs/actors/proxy.h> #include <ydb/core/yq/libs/events/events.h> - -#include <library/cpp/time_provider/time_provider.h> -#include <library/cpp/testing/common/env.h> -#include <library/cpp/protobuf/util/pb_io.h> -#include <library/cpp/actors/testlib/test_runtime.h> - + +#include <library/cpp/time_provider/time_provider.h> +#include <library/cpp/testing/common/env.h> +#include <library/cpp/protobuf/util/pb_io.h> +#include <library/cpp/actors/testlib/test_runtime.h> + #include <ydb/library/yql/utils/bind_in_range.h> - -#include <util/system/file.h> -#include <util/stream/str.h> -#include <util/string/printf.h> -#include <util/string/builder.h> - + +#include <util/system/file.h> +#include <util/stream/str.h> +#include <util/string/printf.h> +#include <util/string/builder.h> + #include <library/cpp/protobuf/json/proto2json.h> -using namespace NYdb; -using namespace NYdb::NYq; -using namespace NActors; - +using namespace NYdb; +using namespace NYdb::NYq; +using namespace NActors; + void UpsertToExistingTable(TDriver& driver, const TString& location){ Y_UNUSED(location); const TString tablePrefix = "Root/yq"; - NYdb::NTable::TClientSettings settings; - NYdb::NTable::TTableClient client(driver, settings); - auto sessionOp = client.CreateSession().ExtractValueSync(); - if (!sessionOp.IsSuccess()) { - ythrow yexception() << sessionOp.GetStatus() << "\n" << sessionOp.GetIssues().ToString(); - } - auto session = sessionOp.GetSession(); - auto timeProvider = CreateDefaultTimeProvider(); - auto now = timeProvider->Now(); - NYdb::TParamsBuilder paramsBuilder; + NYdb::NTable::TClientSettings settings; + NYdb::NTable::TTableClient client(driver, settings); + auto sessionOp = client.CreateSession().ExtractValueSync(); + if (!sessionOp.IsSuccess()) { + ythrow yexception() << sessionOp.GetStatus() << "\n" << sessionOp.GetIssues().ToString(); + } + auto session = sessionOp.GetSession(); + auto timeProvider = CreateDefaultTimeProvider(); + auto now = timeProvider->Now(); + NYdb::TParamsBuilder paramsBuilder; + + paramsBuilder.AddParam("$now").Timestamp(now).Build(); + auto params = paramsBuilder.Build(); - paramsBuilder.AddParam("$now").Timestamp(now).Build(); - auto params = paramsBuilder.Build(); - const TString scope = TScope("some_folder_id").ToString(); - - { - auto result = session.ExecuteSchemeQuery( - Sprintf(R"___( - --!syntax_v1 - CREATE TABLE `%s/%s` ( - id String, - PRIMARY KEY (id) - ); - )___", tablePrefix.c_str(), "empty_table") - ).ExtractValueSync(); - if (!result.IsSuccess()) { - ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); - } - } - - { - NYdb::TParamsBuilder paramsBuilder; - auto result = session.ExecuteDataQuery( - Sprintf(R"___( - --!syntax_v1 - upsert into `%s/empty_table` - (id) - values - ("test_id1"), ("test_id2"); - )___", tablePrefix.c_str()), - NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), - paramsBuilder.Build(), - NYdb::NTable::TExecDataQuerySettings()).ExtractValueSync(); - if (!result.IsSuccess()) { - ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); - } - } - - { - NYdb::TParamsBuilder paramsBuilder; - auto result = session.ExecuteDataQuery( - Sprintf(R"___( - --!syntax_v1 - delete from `%s/empty_table`; - )___", tablePrefix.c_str()), - NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), - paramsBuilder.Build(), - NYdb::NTable::TExecDataQuerySettings()).ExtractValueSync(); - if (!result.IsSuccess()) { - ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); - } - } -} + + { + auto result = session.ExecuteSchemeQuery( + Sprintf(R"___( + --!syntax_v1 + CREATE TABLE `%s/%s` ( + id String, + PRIMARY KEY (id) + ); + )___", tablePrefix.c_str(), "empty_table") + ).ExtractValueSync(); + if (!result.IsSuccess()) { + ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); + } + } + + { + NYdb::TParamsBuilder paramsBuilder; + auto result = session.ExecuteDataQuery( + Sprintf(R"___( + --!syntax_v1 + upsert into `%s/empty_table` + (id) + values + ("test_id1"), ("test_id2"); + )___", tablePrefix.c_str()), + NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), + paramsBuilder.Build(), + NYdb::NTable::TExecDataQuerySettings()).ExtractValueSync(); + if (!result.IsSuccess()) { + ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); + } + } + + { + NYdb::TParamsBuilder paramsBuilder; + auto result = session.ExecuteDataQuery( + Sprintf(R"___( + --!syntax_v1 + delete from `%s/empty_table`; + )___", tablePrefix.c_str()), + NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), + paramsBuilder.Build(), + NYdb::NTable::TExecDataQuerySettings()).ExtractValueSync(); + if (!result.IsSuccess()) { + ythrow yexception() << result.GetStatus() << "\n" << result.GetIssues().ToString(); + } + } +} diff --git a/ydb/services/yq/ut_integration/ut_utils.h b/ydb/services/yq/ut_integration/ut_utils.h index 3dfa09b2bf..a627c45929 100644 --- a/ydb/services/yq/ut_integration/ut_utils.h +++ b/ydb/services/yq/ut_integration/ut_utils.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> -#include <util/system/shellcommand.h> -#include <contrib/libs/curl/include/curl/curl.h> +#include <util/system/shellcommand.h> +#include <contrib/libs/curl/include/curl/curl.h> #include <library/cpp/json/json_reader.h> - + void UpsertToExistingTable(NYdb::TDriver& driver, const TString& location); - + diff --git a/ydb/services/yq/ut_integration/ya.make b/ydb/services/yq/ut_integration/ya.make index 1b24bdb3b6..bc62df4563 100644 --- a/ydb/services/yq/ut_integration/ya.make +++ b/ydb/services/yq/ut_integration/ya.make @@ -11,7 +11,7 @@ FORK_SUBTESTS() SIZE(MEDIUM) SRCS( - ut_utils.cpp + ut_utils.cpp yq_ut.cpp ) diff --git a/ydb/services/yq/ut_integration/yq_ut.cpp b/ydb/services/yq/ut_integration/yq_ut.cpp index f85b79248c..d4a35f195e 100644 --- a/ydb/services/yq/ut_integration/yq_ut.cpp +++ b/ydb/services/yq/ut_integration/yq_ut.cpp @@ -11,13 +11,13 @@ #include <ydb/core/yq/libs/actors/database_resolver.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> - -#include <library/cpp/protobuf/util/pb_io.h> + +#include <library/cpp/protobuf/util/pb_io.h> #include <library/cpp/retry/retry.h> #include <util/system/mutex.h> -#include "ut_utils.h" +#include "ut_utils.h" #include <google/protobuf/util/time_util.h> - + #include <ydb/public/lib/yq/scope.h> #include <util/system/hostname.h> @@ -25,7 +25,7 @@ using namespace NYdb; using namespace YandexQuery;; -using namespace NYdb::NYq; +using namespace NYdb::NYq; namespace { const ui32 Retries = 10; @@ -43,7 +43,7 @@ namespace { TString CreateNewHistoryAndWaitFinish(const TString& folderId, NYdb::NYq::TClient& client, const TString& yqlText, const YandexQuery::QueryMeta::ComputeStatus& expectedStatusResult) - { + { //CreateQuery TString queryId; { @@ -53,7 +53,7 @@ namespace { auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) .ExtractValueSync(); - + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); queryId = result.GetResult().query_id(); } @@ -69,7 +69,7 @@ namespace { return result.GetResult().status() == expectedStatusResult; }, TRetryOptions(Retries)); UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); - + return queryId; } @@ -104,7 +104,7 @@ namespace { } Y_UNIT_TEST_SUITE(Yq_1) { - Y_UNIT_TEST(Basic) { + Y_UNIT_TEST(Basic) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; @@ -196,16 +196,16 @@ Y_UNIT_TEST_SUITE(Yq_1) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); } } - - Y_UNIT_TEST(Basic_EmptyTable) { + + Y_UNIT_TEST(Basic_EmptyTable) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + TString location = TStringBuilder() << "localhost:" << grpc; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - UpsertToExistingTable(driver, location); + UpsertToExistingTable(driver, location); NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - { + const TString folderId = "some_folder_id"; + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("testdbempty") .CreateYdb("Root", location, "") @@ -214,96 +214,96 @@ Y_UNIT_TEST_SUITE(Yq_1) { .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - + } + const TString queryId = CreateNewHistoryAndWaitFinish( folderId, client, "select count(*) from testdbempty.`yq/empty_table`", YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 0); - } - - Y_UNIT_TEST(Basic_EmptyList) { + } + + Y_UNIT_TEST(Basic_EmptyList) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + TString location = TStringBuilder() << "localhost:" << grpc; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; - CreateNewHistoryAndWaitFinish(folderId, client, "select []", expectedStatus); - } - - Y_UNIT_TEST(Basic_EmptyDict) { + const TString folderId = "some_folder_id"; + auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; + CreateNewHistoryAndWaitFinish(folderId, client, "select []", expectedStatus); + } + + Y_UNIT_TEST(Basic_EmptyDict) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; - CreateNewHistoryAndWaitFinish(folderId, client, "select {}", expectedStatus); - } - - Y_UNIT_TEST(Basic_Null) { + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const TString folderId = "some_folder_id"; + auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; + CreateNewHistoryAndWaitFinish(folderId, client, "select {}", expectedStatus); + } + + Y_UNIT_TEST(Basic_Null) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; - CreateNewHistoryAndWaitFinish(folderId, client, "select null", expectedStatus); - } - + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const TString folderId = "some_folder_id"; + auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; + CreateNewHistoryAndWaitFinish(folderId, client, "select null", expectedStatus); + } + SIMPLE_UNIT_FORKED_TEST(Basic_Tagged) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - - - { - auto request = ::NYq::TCreateConnectionBuilder{} - .SetName("testdb00") - .CreateYdb("Root", location, "") - .Build(); - - auto result = client.CreateConnection( - request, CreateYqSettings<TCreateConnectionSettings>(folderId)) - .ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; - CreateNewHistoryAndWaitFinish(folderId, client, "select AsTagged(count(*), \"tag\") from testdb00.`yq/connections`", expectedStatus); - } - - Y_UNIT_TEST(Basic_TaggedLiteral) { + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const TString folderId = "some_folder_id"; + + + { + auto request = ::NYq::TCreateConnectionBuilder{} + .SetName("testdb00") + .CreateYdb("Root", location, "") + .Build(); + + auto result = client.CreateConnection( + request, CreateYqSettings<TCreateConnectionSettings>(folderId)) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; + CreateNewHistoryAndWaitFinish(folderId, client, "select AsTagged(count(*), \"tag\") from testdb00.`yq/connections`", expectedStatus); + } + + Y_UNIT_TEST(Basic_TaggedLiteral) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); - ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - NYdb::NYq::TClient client(driver); - const TString folderId = "some_folder_id"; - - auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; - CreateNewHistoryAndWaitFinish(folderId, client, "select AsTagged(1, \"tag\")", expectedStatus); - } - - // use fork for data test due to ch initialization problem + ui16 grpc = server.GetPort(); + TString location = TStringBuilder() << "localhost:" << grpc; + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); + NYdb::NYq::TClient client(driver); + const TString folderId = "some_folder_id"; + + auto expectedStatus = YandexQuery::QueryMeta::COMPLETED; + CreateNewHistoryAndWaitFinish(folderId, client, "select AsTagged(1, \"tag\")", expectedStatus); + } + + // use fork for data test due to ch initialization problem SIMPLE_UNIT_FORKED_TEST(ExtendedDatabaseId) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + TString location = TStringBuilder() << "localhost:" << grpc; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); - + NYdb::NYq::TClient client(driver); - const TString folderId = "folder_id_" + CreateGuidAsString(); - { + const TString folderId = "folder_id_" + CreateGuidAsString(); + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("testdb01") .CreateYdb("FakeDatabaseId", "") @@ -312,9 +312,9 @@ Y_UNIT_TEST_SUITE(Yq_1) { .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { + } + + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("testdb02") .CreateYdb("FakeDatabaseId", "") @@ -323,8 +323,8 @@ Y_UNIT_TEST_SUITE(Yq_1) { .CreateConnection(request, CreateYqSettings<TCreateConnectionSettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); - } - + } + { const auto queryId = CreateNewHistoryAndWaitFinish(folderId, client, "select count(*) from testdb01.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); @@ -337,12 +337,12 @@ Y_UNIT_TEST_SUITE(Yq_1) { "select count(*) from testdb02.`yq/connections`", YandexQuery::QueryMeta::COMPLETED); CheckGetResultData(client, queryId, folderId, 1, 1, 2); } - } - + } + Y_UNIT_TEST(DescribeConnection) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + TString location = TStringBuilder() << "localhost:" << grpc; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken("root@builtin")); NYdb::NYq::TClient client(driver); const TString folderId = "some_folder_id"; @@ -373,7 +373,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { UNIT_ASSERT_VALUES_EQUAL(res.content().name(), "created_conn"); } } - + Y_UNIT_TEST(ListConnections) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); @@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); } - { + { const auto request = ::NYq::TListConnectionsBuilder().Build(); auto result = client .ListConnections(request, CreateYqSettings<TListConnectionsSettings>(folderId)) @@ -431,8 +431,8 @@ Y_UNIT_TEST_SUITE(Yq_1) { } i++; } - } - } + } + } Y_UNIT_TEST(ListConnectionsOnEmptyConnectionsTable) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); @@ -461,7 +461,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { NYdb::NYq::TClient client(driver); const auto folderId = TString(__func__) + "folder_id"; TString conId; - + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("created_conn") @@ -473,7 +473,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); conId = result.GetResult().connection_id(); } - + {//Modify const auto request = ::NYq::TModifyConnectionBuilder() .SetName("modified_name") @@ -486,7 +486,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { .ExtractValueSync(); UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); } - + { auto request = ::NYq::TDescribeConnectionBuilder() .SetConnectionId(conId) @@ -512,7 +512,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { NYdb::NYq::TClient client(driver); const auto folderId = TString(__func__) + "folder_id"; TString conId; - + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("created_conn") @@ -542,12 +542,12 @@ Y_UNIT_TEST_SUITE(Yq_1) { ui16 grpc = server.GetPort(); TString location = TStringBuilder() << "localhost:" << grpc; TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); NYdb::NYq::TClient client(driver); - + const auto folderId = TString(__func__) + "folder_id"; TString conId; - + { const auto request = ::NYq::TCreateConnectionBuilder() .SetName("created_conn") @@ -559,7 +559,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { UNIT_ASSERT_C(result.GetStatus() == EStatus::SUCCESS, result.GetIssues().ToString()); conId = result.GetResult().connection_id(); } - + { const auto request = ::NYq::TModifyConnectionBuilder() .SetConnectionId(conId) @@ -650,29 +650,29 @@ Y_UNIT_TEST_SUITE(Yq_1) { Y_UNIT_TEST(CreateQuery_With_Idempotency) {//TODO Fix TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); - TString location = TStringBuilder() << "localhost:" << grpc; + TString location = TStringBuilder() << "localhost:" << grpc; TString userToken = "root@builtin"; - auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); + auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); NYdb::NYq::TClient client(driver); - + const auto folderId = TString(__func__) + "folder_id"; - const TString idempotencyKey = "idempotency_key"; + const TString idempotencyKey = "idempotency_key"; const TString yqlText = "select 1"; TString queryId; const auto request = ::NYq::TCreateQueryBuilder{} .SetText(yqlText) .SetIdempotencyKey(idempotencyKey) .Build(); - - { + + { auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); queryId = result.GetResult().query_id(); - } - - { + } + + { const auto req = ::NYq::TDescribeQueryBuilder{} .SetQueryId(queryId) .Build(); @@ -686,19 +686,19 @@ Y_UNIT_TEST_SUITE(Yq_1) { return status == YandexQuery::QueryMeta::COMPLETED; }, TRetryOptions(Retries)); UNIT_ASSERT_C(result, "the execution of the query did not end within the time limit"); - } - { + } + { auto result = client.CreateQuery( request, CreateYqSettings<TCreateQuerySettings>(folderId)) .ExtractValueSync(); - + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(queryId, result.GetResult().query_id()); - } + } CheckGetResultData(client, queryId, folderId, 1, 1, 1); - } - - // use fork for data test due to ch initialization problem + } + + // use fork for data test due to ch initialization problem SIMPLE_UNIT_FORKED_TEST(CreateQuery_Without_Connection) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); @@ -719,7 +719,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { TString userToken = "root@builtin"; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); NYdb::NYq::TClient client(driver); - + const auto folderId = TString(__func__) + "folder_id"; const TString yqlText = "select 1"; const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client, @@ -755,7 +755,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { TString userToken = "root@builtin"; auto driver = TDriver(TDriverConfig().SetEndpoint(location).SetAuthToken(userToken)); NYdb::NYq::TClient client(driver); - + const auto folderId = TString(__func__) + "folder_id"; const TString yqlText = "select 1"; const TString queryId = CreateNewHistoryAndWaitFinish(folderId, client, @@ -856,7 +856,7 @@ Y_UNIT_TEST_SUITE(Yq_1) { } Y_UNIT_TEST_SUITE(Yq_2) { - // use fork for data test due to ch initialization problem + // use fork for data test due to ch initialization problem Y_UNIT_TEST(Test_HostNameTrasformation) { UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("rc1c-p5waby2y5y1kb5ue.mdb.yandexcloud.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443"); UNIT_ASSERT_VALUES_EQUAL(::NYq::TransformMdbHostToCorrectFormat("xxx.xxx"), "xxx.db.yandex.net:8443"); @@ -924,7 +924,7 @@ Y_UNIT_TEST_SUITE(Yq_2) { UNIT_ASSERT_VALUES_EQUAL(resultSet.rows(0).items(0).uint64_value(), 1); } } -} +} Y_UNIT_TEST_SUITE(PrivateApi) { Y_UNIT_TEST(PingTask) { |