1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
#include <Backups/BackupIO_File.h>
#include <Disks/DiskLocal.h>
#include <Disks/IO/createReadBufferFromFileBase.h>
#include <IO/WriteBufferFromFile.h>
#include <Common/logger_useful.h>
namespace fs = std::filesystem;
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
BackupReaderFile::BackupReaderFile(const String & root_path_, const ReadSettings & read_settings_, const WriteSettings & write_settings_)
: BackupReaderDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupReaderFile"))
, root_path(root_path_)
, data_source_description(DiskLocal::getLocalDataSourceDescription(root_path))
{
}
bool BackupReaderFile::fileExists(const String & file_name)
{
return fs::exists(root_path / file_name);
}
UInt64 BackupReaderFile::getFileSize(const String & file_name)
{
return fs::file_size(root_path / file_name);
}
std::unique_ptr<SeekableReadBuffer> BackupReaderFile::readFile(const String & file_name)
{
return createReadBufferFromFileBase(root_path / file_name, read_settings);
}
void BackupReaderFile::copyFileToDisk(const String & path_in_backup, size_t file_size, bool encrypted_in_backup,
DiskPtr destination_disk, const String & destination_path, WriteMode write_mode)
{
/// std::filesystem::copy() can copy from the filesystem only, and can't do throttling or appending.
bool has_throttling = static_cast<bool>(read_settings.local_throttler);
if (!has_throttling && (write_mode == WriteMode::Rewrite))
{
auto destination_data_source_description = destination_disk->getDataSourceDescription();
if (destination_data_source_description.sameKind(data_source_description)
&& (destination_data_source_description.is_encrypted == encrypted_in_backup))
{
/// Use more optimal way.
LOG_TRACE(log, "Copying file {} to disk {} locally", path_in_backup, destination_disk->getName());
auto write_blob_function = [abs_source_path = root_path / path_in_backup, file_size](
const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
{
/// For local disks the size of a blob path is expected to be 1.
if (blob_path.size() != 1 || mode != WriteMode::Rewrite)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Blob writing function called with unexpected blob_path.size={} or mode={}",
blob_path.size(), mode);
fs::copy(abs_source_path, blob_path.at(0), fs::copy_options::overwrite_existing);
return file_size;
};
destination_disk->writeFileUsingBlobWritingFunction(destination_path, write_mode, write_blob_function);
return; /// copied!
}
}
/// Fallback to copy through buffers.
BackupReaderDefault::copyFileToDisk(path_in_backup, file_size, encrypted_in_backup, destination_disk, destination_path, write_mode);
}
BackupWriterFile::BackupWriterFile(const String & root_path_, const ReadSettings & read_settings_, const WriteSettings & write_settings_)
: BackupWriterDefault(read_settings_, write_settings_, &Poco::Logger::get("BackupWriterFile"))
, root_path(root_path_)
, data_source_description(DiskLocal::getLocalDataSourceDescription(root_path))
{
}
bool BackupWriterFile::fileExists(const String & file_name)
{
return fs::exists(root_path / file_name);
}
UInt64 BackupWriterFile::getFileSize(const String & file_name)
{
return fs::file_size(root_path / file_name);
}
std::unique_ptr<ReadBuffer> BackupWriterFile::readFile(const String & file_name, size_t expected_file_size)
{
return createReadBufferFromFileBase(root_path / file_name, read_settings.adjustBufferSize(expected_file_size));
}
std::unique_ptr<WriteBuffer> BackupWriterFile::writeFile(const String & file_name)
{
auto file_path = root_path / file_name;
fs::create_directories(file_path.parent_path());
return std::make_unique<WriteBufferFromFile>(file_path, write_buffer_size, -1, write_settings.local_throttler);
}
void BackupWriterFile::removeFile(const String & file_name)
{
fs::remove(root_path / file_name);
if (fs::is_directory(root_path) && fs::is_empty(root_path))
fs::remove(root_path);
}
void BackupWriterFile::removeFiles(const Strings & file_names)
{
for (const auto & file_name : file_names)
fs::remove(root_path / file_name);
if (fs::is_directory(root_path) && fs::is_empty(root_path))
fs::remove(root_path);
}
void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
bool copy_encrypted, UInt64 start_pos, UInt64 length)
{
/// std::filesystem::copy() can copy from the filesystem only, and can't do throttling or copy a part of the file.
bool has_throttling = static_cast<bool>(read_settings.local_throttler);
if (!has_throttling)
{
auto source_data_source_description = src_disk->getDataSourceDescription();
if (source_data_source_description.sameKind(data_source_description)
&& (source_data_source_description.is_encrypted == copy_encrypted))
{
/// std::filesystem::copy() can copy from a single file only.
if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 1)
{
auto abs_source_path = blob_path[0];
/// std::filesystem::copy() can copy a file as a whole only.
if ((start_pos == 0) && (length == fs::file_size(abs_source_path)))
{
/// Use more optimal way.
LOG_TRACE(log, "Copying file {} from disk {} locally", src_path, src_disk->getName());
auto abs_dest_path = root_path / path_in_backup;
fs::create_directories(abs_dest_path.parent_path());
fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing);
return; /// copied!
}
}
}
}
/// Fallback to copy through buffers.
BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length);
}
}
|
