1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
|
#include <Databases/DatabaseFilesystem.h>
#include <IO/Operators.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/parseQuery.h>
#include <Storages/IStorage.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Common/filesystemHelpers.h>
#include <Formats/FormatFactory.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int UNKNOWN_TABLE;
extern const int PATH_ACCESS_DENIED;
extern const int BAD_ARGUMENTS;
extern const int FILE_DOESNT_EXIST;
}
DatabaseFilesystem::DatabaseFilesystem(const String & name_, const String & path_, ContextPtr context_)
: IDatabase(name_), WithContext(context_->getGlobalContext()), path(path_), log(&Poco::Logger::get("DatabaseFileSystem(" + name_ + ")"))
{
bool is_local = context_->getApplicationType() == Context::ApplicationType::LOCAL;
fs::path user_files_path = is_local ? "" : fs::canonical(getContext()->getUserFilesPath());
if (fs::path(path).is_relative())
{
path = user_files_path / path;
}
else if (!is_local && !pathStartsWith(fs::path(path), user_files_path))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Path must be inside user-files path: {}", user_files_path.string());
}
path = fs::absolute(path).lexically_normal();
if (!fs::exists(path))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path does not exist: {}", path);
}
std::string DatabaseFilesystem::getTablePath(const std::string & table_name) const
{
fs::path table_path = fs::path(path) / table_name;
return table_path.lexically_normal().string();
}
void DatabaseFilesystem::addTable(const std::string & table_name, StoragePtr table_storage) const
{
std::lock_guard lock(mutex);
auto [_, inserted] = loaded_tables.emplace(table_name, table_storage);
if (!inserted)
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Table with name `{}` already exists in database `{}` (engine {})",
table_name, getDatabaseName(), getEngineName());
}
bool DatabaseFilesystem::checkTableFilePath(const std::string & table_path, ContextPtr context_, bool throw_on_error) const
{
/// If run in Local mode, no need for path checking.
bool check_path = context_->getApplicationType() != Context::ApplicationType::LOCAL;
const auto & user_files_path = context_->getUserFilesPath();
/// Check access for file before checking its existence.
if (check_path && !fileOrSymlinkPathStartsWith(table_path, user_files_path))
{
/// Access denied is thrown regardless of 'throw_on_error'
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File is not inside {}", user_files_path);
}
if (!containsGlobs(table_path))
{
/// Check if the corresponding file exists.
if (!fs::exists(table_path))
{
if (throw_on_error)
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File does not exist: {}", table_path);
else
return false;
}
if (!fs::is_regular_file(table_path))
{
if (throw_on_error)
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File is directory, but expected a file: {}", table_path);
else
return false;
}
}
return true;
}
StoragePtr DatabaseFilesystem::tryGetTableFromCache(const std::string & name) const
{
StoragePtr table = nullptr;
{
std::lock_guard lock(mutex);
auto it = loaded_tables.find(name);
if (it != loaded_tables.end())
table = it->second;
}
/// Invalidate cache if file no longer exists.
if (table && !fs::exists(getTablePath(name)))
{
std::lock_guard lock(mutex);
loaded_tables.erase(name);
return nullptr;
}
return table;
}
bool DatabaseFilesystem::isTableExist(const String & name, ContextPtr context_) const
{
if (tryGetTableFromCache(name))
return true;
return checkTableFilePath(getTablePath(name), context_, /* throw_on_error */ false);
}
StoragePtr DatabaseFilesystem::getTableImpl(const String & name, ContextPtr context_, bool throw_on_error) const
{
/// Check if table exists in loaded tables map.
if (auto table = tryGetTableFromCache(name))
return table;
auto table_path = getTablePath(name);
if (!checkTableFilePath(table_path, context_, throw_on_error))
return {};
auto format = FormatFactory::instance().getFormatFromFileName(table_path, throw_on_error);
if (format.empty())
return {};
auto ast_function_ptr = makeASTFunction("file", std::make_shared<ASTLiteral>(table_path), std::make_shared<ASTLiteral>(format));
auto table_function = TableFunctionFactory::instance().get(ast_function_ptr, context_);
if (!table_function)
return nullptr;
/// TableFunctionFile throws exceptions, if table cannot be created.
auto table_storage = table_function->execute(ast_function_ptr, context_, name);
if (table_storage)
addTable(name, table_storage);
return table_storage;
}
StoragePtr DatabaseFilesystem::getTable(const String & name, ContextPtr context_) const
{
/// getTableImpl can throw exceptions, do not catch them to show correct error to user.
if (auto storage = getTableImpl(name, context_, true))
return storage;
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name));
}
StoragePtr DatabaseFilesystem::tryGetTable(const String & name, ContextPtr context_) const
{
return getTableImpl(name, context_, false);
}
bool DatabaseFilesystem::empty() const
{
std::lock_guard lock(mutex);
return loaded_tables.empty();
}
ASTPtr DatabaseFilesystem::getCreateDatabaseQuery() const
{
const auto & settings = getContext()->getSettingsRef();
const String query = fmt::format("CREATE DATABASE {} ENGINE = Filesystem('{}')", backQuoteIfNeed(getDatabaseName()), path);
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
if (const auto database_comment = getDatabaseComment(); !database_comment.empty())
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.set(ast_create_query.comment, std::make_shared<ASTLiteral>(database_comment));
}
return ast;
}
void DatabaseFilesystem::shutdown()
{
Tables tables_snapshot;
{
std::lock_guard lock(mutex);
tables_snapshot = loaded_tables;
}
for (const auto & kv : tables_snapshot)
{
auto table_id = kv.second->getStorageID();
kv.second->flushAndShutdown();
}
std::lock_guard lock(mutex);
loaded_tables.clear();
}
/**
* Returns an empty vector because the database is read-only and no tables can be backed up
*/
std::vector<std::pair<ASTPtr, StoragePtr>> DatabaseFilesystem::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const
{
return {};
}
/**
*
* Returns an empty iterator because the database does not have its own tables
* But only caches them for quick access
*/
DatabaseTablesIteratorPtr DatabaseFilesystem::getTablesIterator(ContextPtr, const FilterByNameFunction &) const
{
return std::make_unique<DatabaseTablesSnapshotIterator>(Tables{}, getDatabaseName());
}
}
|
