1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
#include "yql_s3_settings.h"
#include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h>
#include <util/generic/size_literals.h>
namespace NYql {
using namespace NCommon;
TS3Configuration::TS3Configuration()
{
REGISTER_SETTING(*this, SourceCoroActor);
REGISTER_SETTING(*this, MaxOutputObjectSize);
REGISTER_SETTING(*this, UniqueKeysCountLimit);
REGISTER_SETTING(*this, BlockSizeMemoryLimit);
REGISTER_SETTING(*this, SerializeMemoryLimit);
REGISTER_SETTING(*this, InFlightMemoryLimit);
REGISTER_SETTING(*this, JsonListSizeLimit).Upper(100'000);
REGISTER_SETTING(*this, ArrowThreadPool);
REGISTER_SETTING(*this, ArrowParallelRowGroupCount).Lower(1);
REGISTER_SETTING(*this, ArrowRowGroupReordering);
}
TS3Settings::TConstPtr TS3Configuration::Snapshot() const {
return std::make_shared<const TS3Settings>(*this);
}
bool TS3Configuration::HasCluster(TStringBuf cluster) const {
return ValidClusters.contains(cluster);
}
void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeAnnotationContext> typeCtx)
{
for (auto& formatSizeLimit: config.GetFormatSizeLimit()) {
if (formatSizeLimit.GetName()) { // ignore unnamed limits
FormatSizeLimits.emplace(formatSizeLimit.GetName(), formatSizeLimit.GetFileSizeLimit());
}
}
FileSizeLimit = config.HasFileSizeLimit() ? config.GetFileSizeLimit() : 2_GB;
MaxFilesPerQuery = config.HasMaxFilesPerQuery() ? config.GetMaxFilesPerQuery() : 7000;
MaxDiscoveryFilesPerQuery = config.HasMaxDiscoveryFilesPerQuery() ? config.GetMaxDiscoveryFilesPerQuery() : 9000;
MaxDirectoriesAndFilesPerQuery = config.HasMaxDirectoriesAndFilesPerQuery() ? config.GetMaxDirectoriesAndFilesPerQuery() : 9000;
MinDesiredDirectoriesOfFilesPerQuery = config.HasMinDesiredDirectoriesOfFilesPerQuery() ? config.GetMinDesiredDirectoriesOfFilesPerQuery() : 100;
MaxReadSizePerQuery = config.HasMaxReadSizePerQuery() ? config.GetMaxReadSizePerQuery() : 4_GB;
MaxInflightListsPerQuery = config.HasMaxInflightListsPerQuery() ? config.GetMaxInflightListsPerQuery() : 1;
TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
for (auto& cluster: config.GetClusterMapping()) {
clusters.push_back(cluster.GetName());
}
this->SetValidClusters(clusters);
this->Dispatch(config.GetDefaultSettings());
for (const auto& cluster: config.GetClusterMapping()) {
this->Dispatch(cluster.GetName(), cluster.GetSettings());
auto& settings = Clusters[cluster.GetName()];
settings.Url = cluster.GetUrl();
TString authToken;
if (const auto& token = cluster.GetToken()) {
authToken = typeCtx->Credentials->FindCredentialContent("cluster:default_" + cluster.GetName(), "", token);
}
Tokens[cluster.GetName()] = ComposeStructuredTokenJsonForServiceAccount(cluster.GetServiceAccountId(), cluster.GetServiceAccountIdSignature(), authToken);
}
this->FreezeDefaults();
}
} // NYql
|
