blob: 5fb9dd907a174f5a9dfa6c8ad08cb672a6cf5175 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
#pragma once
#include <Formats/NativeReader.h>
#include <Formats/NativeWriter.h>
#include <Core/BackgroundSchedulePool.h>
#include <Disks/IDisk.h>
#include <Storages/MergeTree/IMergeTreeDataPart.h>
#include <Storages/MergeTree/MergeTreeDataPartInMemory.h>
namespace DB
{
class MergeTreeData;
struct DataPartsLock;
/** WAL stores addditions and removals of data parts in in-memory format.
* Format of data in WAL:
* - version
* - type of action (ADD or DROP)
* - part name
* - part's block in Native format. (for ADD action)
*/
class MergeTreeWriteAheadLog
{
public:
/// Append-only enum. It is serialized to WAL
enum class ActionType : UInt8
{
ADD_PART = 0,
DROP_PART = 1,
};
struct ActionMetadata
{
/// The minimum version of WAL reader that can understand metadata written by current ClickHouse version.
/// This field must be increased when making backwards incompatible changes.
///
/// The same approach can be used recursively inside metadata.
UInt8 min_compatible_version = 0;
/// Actual metadata.
UUID part_uuid = UUIDHelpers::Nil;
void write(WriteBuffer & meta_out) const;
void read(ReadBuffer & meta_in);
private:
static constexpr auto JSON_KEY_PART_UUID = "part_uuid";
String toJSON() const;
void fromJSON(const String & buf);
};
constexpr static UInt8 WAL_VERSION = 1;
constexpr static auto WAL_FILE_NAME = "wal";
constexpr static auto WAL_FILE_EXTENSION = ".bin";
constexpr static auto DEFAULT_WAL_FILE_NAME = "wal.bin";
MergeTreeWriteAheadLog(MergeTreeData & storage_, const DiskPtr & disk_,
const String & name = DEFAULT_WAL_FILE_NAME);
~MergeTreeWriteAheadLog();
void dropPart(const String & part_name);
std::vector<MergeTreeMutableDataPartPtr> restore(
const StorageMetadataPtr & metadata_snapshot,
ContextPtr context,
DataPartsLock & parts_lock,
bool readonly);
using MinMaxBlockNumber = std::pair<Int64, Int64>;
static std::optional<MinMaxBlockNumber> tryParseMinMaxBlockNumber(const String & filename);
void shutdown();
/// Drop all write ahead logs from disk. Useful during table drop.
static void dropAllWriteAheadLogs(DiskPtr disk_to_drop, std::string relative_data_path);
private:
void init();
void rotate(const std::unique_lock<std::mutex> & lock);
const MergeTreeData & storage;
DiskPtr disk;
String name;
String path;
std::unique_ptr<WriteBuffer> out;
std::unique_ptr<NativeWriter> block_out;
Int64 min_block_number = std::numeric_limits<Int64>::max();
Int64 max_block_number = -1;
BackgroundSchedulePool & pool;
BackgroundSchedulePoolTaskHolder sync_task;
std::condition_variable sync_cv;
size_t bytes_at_last_sync = 0;
bool sync_scheduled = false;
bool shutdown_called = false;
mutable std::mutex write_mutex;
Poco::Logger * log;
};
}
|
