1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
#include "RegionsNames.h"
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Poco/Exception.h>
#include <Common/logger_useful.h>
#include "GeodataProviders/INamesProvider.h"
namespace DB
{
namespace ErrorCodes
{
extern const int INCORRECT_DATA;
}
}
RegionsNames::RegionsNames(IRegionsNamesDataProviderPtr data_provider)
{
for (size_t language_id = 0; language_id < total_languages; ++language_id)
{
const std::string & language = languages[language_id];
names_sources[language_id] = data_provider->getLanguageRegionsNamesSource(language);
}
reload();
}
std::string RegionsNames::dumpSupportedLanguagesNames()
{
DB::WriteBufferFromOwnString out;
for (size_t i = 0; i < total_languages; ++i)
{
if (i > 0)
out << ", ";
out << '\'' << languages[i] << '\'';
}
return out.str();
}
void RegionsNames::reload()
{
Poco::Logger * log = &Poco::Logger::get("RegionsNames");
LOG_DEBUG(log, "Reloading regions names");
RegionID max_region_id = 0;
for (size_t language_id = 0; language_id < total_languages; ++language_id)
{
const std::string & language = languages[language_id];
auto names_source = names_sources[language_id];
if (!names_source || !names_source->isModified())
continue;
LOG_DEBUG(log, "Reloading regions names for language: {}", language);
auto names_reader = names_source->createReader();
const size_t initial_size = 10000;
const size_t max_size = 15000000;
Chars new_chars;
StringRefs new_names_refs(initial_size, StringRef("", 0));
/// Allocate a continuous slice of memory, which is enough to store all names.
new_chars.reserve(names_source->estimateTotalSize());
RegionNameEntry name_entry;
while (names_reader->readNext(name_entry))
{
size_t old_size = new_chars.size();
if (new_chars.capacity() < old_size + name_entry.name.length() + 1)
throw Poco::Exception("Logical error. Maybe size estimate of " + names_source->getSourceName() + " is wrong.");
new_chars.resize(old_size + name_entry.name.length() + 1);
memcpy(new_chars.data() + old_size, name_entry.name.c_str(), name_entry.name.length() + 1);
if (name_entry.id > max_region_id)
{
max_region_id = name_entry.id;
if (name_entry.id > max_size)
throw DB::Exception(DB::ErrorCodes::INCORRECT_DATA,
"Region id is too large: {}, should be not more than {}",
DB::toString(name_entry.id), DB::toString(max_size));
}
while (name_entry.id >= new_names_refs.size())
new_names_refs.resize(new_names_refs.size() * 2, StringRef("", 0));
new_names_refs[name_entry.id] = StringRef(new_chars.data() + old_size, name_entry.name.length());
}
chars[language_id].swap(new_chars);
names_refs[language_id].swap(new_names_refs);
}
for (size_t language_id = 0; language_id < total_languages; ++language_id)
names_refs[language_id].resize(max_region_id + 1, StringRef("", 0));
}
|
