1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
#include "MetadataStorageFromPlainObjectStorage.h"
#include <Disks/IDisk.h>
#include <Disks/ObjectStorages/StaticDirectoryIterator.h>
#include <Common/filesystemHelpers.h>
#include <Common/logger_useful.h>
#include <Common/StringUtils/StringUtils.h>
#include <IO/WriteHelpers.h>
namespace DB
{
MetadataStorageFromPlainObjectStorage::MetadataStorageFromPlainObjectStorage(
ObjectStoragePtr object_storage_,
const std::string & object_storage_root_path_)
: object_storage(object_storage_)
, object_storage_root_path(object_storage_root_path_)
{
}
MetadataTransactionPtr MetadataStorageFromPlainObjectStorage::createTransaction()
{
return std::make_shared<MetadataStorageFromPlainObjectStorageTransaction>(*this);
}
const std::string & MetadataStorageFromPlainObjectStorage::getPath() const
{
return object_storage_root_path;
}
std::filesystem::path MetadataStorageFromPlainObjectStorage::getAbsolutePath(const std::string & path) const
{
return fs::path(object_storage_root_path) / path;
}
bool MetadataStorageFromPlainObjectStorage::exists(const std::string & path) const
{
/// NOTE: exists() cannot be used here since it works only for existing
/// key, and does not work for some intermediate path.
std::string abs_path = getAbsolutePath(path);
return object_storage->existsOrHasAnyChild(abs_path);
}
bool MetadataStorageFromPlainObjectStorage::isFile(const std::string & path) const
{
/// NOTE: This check is inaccurate and has excessive API calls
return exists(path) && !isDirectory(path);
}
bool MetadataStorageFromPlainObjectStorage::isDirectory(const std::string & path) const
{
std::string directory = getAbsolutePath(path);
if (!directory.ends_with('/'))
directory += '/';
RelativePathsWithMetadata files;
object_storage->listObjects(directory, files, 1);
return !files.empty();
}
uint64_t MetadataStorageFromPlainObjectStorage::getFileSize(const String & path) const
{
RelativePathsWithMetadata children;
auto metadata = object_storage->tryGetObjectMetadata(getAbsolutePath(path));
if (metadata)
return metadata->size_bytes;
return 0;
}
std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(const std::string & path) const
{
RelativePathsWithMetadata files;
std::string abs_path = getAbsolutePath(path);
if (!abs_path.ends_with('/'))
abs_path += '/';
object_storage->listObjects(abs_path, files, 0);
std::vector<std::string> result;
for (const auto & path_size : files)
{
result.push_back(path_size.relative_path);
}
std::unordered_set<std::string> duplicates_filter;
for (auto & row : result)
{
chassert(row.starts_with(abs_path));
row.erase(0, abs_path.size());
auto slash_pos = row.find_first_of('/');
if (slash_pos != std::string::npos)
row.erase(slash_pos, row.size() - slash_pos);
duplicates_filter.insert(row);
}
return std::vector<std::string>(duplicates_filter.begin(), duplicates_filter.end());
}
DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(const std::string & path) const
{
/// Required for MergeTree
auto paths = listDirectory(path);
std::vector<std::filesystem::path> fs_paths(paths.begin(), paths.end());
return std::make_unique<StaticDirectoryIterator>(std::move(fs_paths));
}
StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std::string & path) const
{
std::string blob_name = object_storage->generateBlobNameForPath(path);
size_t object_size = getFileSize(blob_name);
auto object = StoredObject(getAbsolutePath(blob_name), object_size, path);
return {std::move(object)};
}
const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
{
return metadata_storage;
}
void MetadataStorageFromPlainObjectStorageTransaction::unlinkFile(const std::string & path)
{
auto object = StoredObject(metadata_storage.getAbsolutePath(path));
metadata_storage.object_storage->removeObject(object);
}
void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std::string &)
{
/// Noop. It is an Object Storage not a filesystem.
}
void MetadataStorageFromPlainObjectStorageTransaction::createDirectoryRecursive(const std::string &)
{
/// Noop. It is an Object Storage not a filesystem.
}
void MetadataStorageFromPlainObjectStorageTransaction::addBlobToMetadata(
const std::string &, const std::string & /* blob_name */, uint64_t /* size_in_bytes */)
{
/// Noop, local metadata files is only one file, it is the metadata file itself.
}
UnlinkMetadataFileOperationOutcomePtr MetadataStorageFromPlainObjectStorageTransaction::unlinkMetadata(const std::string &)
{
/// No hardlinks, so will always remove file.
return std::make_shared<UnlinkMetadataFileOperationOutcome>(UnlinkMetadataFileOperationOutcome{0});
}
}
|
