1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
#pragma once
#include <Disks/IStoragePolicy.h>
#include <Disks/DiskSelector.h>
#include <Disks/IDisk.h>
#include <Disks/IVolume.h>
#include <Disks/VolumeJBOD.h>
#include <Disks/SingleDiskVolume.h>
#include <IO/WriteHelpers.h>
#include <Common/CurrentMetrics.h>
#include <Common/Exception.h>
#include <Common/formatReadable.h>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <unistd.h>
#include <boost/noncopyable.hpp>
#include <Poco/Util/AbstractConfiguration.h>
namespace DB
{
/**
* Contains all information about volumes configuration for Storage.
* Can determine appropriate Volume and Disk for each reservation.
*/
class StoragePolicy : public IStoragePolicy
{
public:
StoragePolicy(String name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks);
StoragePolicy(String name_, Volumes volumes_, double move_factor_);
StoragePolicy(
StoragePolicyPtr storage_policy,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
DiskSelectorPtr disks
);
bool isDefaultPolicy() const override;
/// Returns disks ordered by volumes priority
Disks getDisks() const override;
/// Returns any disk
/// Used when it's not important, for example for
/// mutations files
DiskPtr getAnyDisk() const override;
DiskPtr tryGetDiskByName(const String & disk_name) const override;
/// Get free space from most free disk
UInt64 getMaxUnreservedFreeSpace() const override;
const String & getName() const override{ return name; }
/// Returns valid reservation or nullptr
ReservationPtr reserve(UInt64 bytes) const override;
/// Reserves space on any volume or throws
ReservationPtr reserveAndCheck(UInt64 bytes) const override;
/// Reserves space on any volume with index > min_volume_index or returns nullptr
ReservationPtr reserve(UInt64 bytes, size_t min_volume_index) const override;
/// Find volume index, which contains disk
std::optional<size_t> tryGetVolumeIndexByDiskName(const String & disk_name) const override;
/// Reserves 0 bytes on disk with max available space
/// Do not use this function when it is possible to predict size.
ReservationPtr makeEmptyReservationOnLargestDisk() const override;
const Volumes & getVolumes() const override{ return volumes; }
/// Returns number [0., 1.] -- fraction of free space on disk
/// which should be kept with help of background moves
double getMoveFactor() const override{ return move_factor; }
/// Get volume by index.
VolumePtr getVolume(size_t index) const override;
VolumePtr tryGetVolumeByName(const String & volume_name) const override;
/// Checks if storage policy can be replaced by another one.
void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override;
/// Check if we have any volume with stopped merges
bool hasAnyVolumeWithDisabledMerges() const override;
bool containsVolume(const String & volume_name) const override;
private:
Volumes volumes;
const String name;
std::unordered_map<String, size_t> volume_index_by_volume_name;
std::unordered_map<String, size_t> volume_index_by_disk_name;
/// move_factor from interval [0., 1.]
/// We move something if disk from this policy
/// filled more than total_size * move_factor
double move_factor = 0.1; /// by default move factor is 10%
void buildVolumeIndices();
Poco::Logger * log;
};
class StoragePolicySelector;
using StoragePolicySelectorPtr = std::shared_ptr<const StoragePolicySelector>;
using StoragePoliciesMap = std::map<String, StoragePolicyPtr>;
/// Parse .xml configuration and store information about policies
/// Mostly used for introspection.
class StoragePolicySelector
{
public:
static constexpr auto TMP_STORAGE_POLICY_PREFIX = "__";
StoragePolicySelector(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks);
StoragePolicySelectorPtr updateFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks) const;
/// Policy by name
StoragePolicyPtr get(const String & name) const;
StoragePolicyPtr tryGet(const String & name) const;
/// All policies
const StoragePoliciesMap & getPoliciesMap() const { return policies; }
/// Add storage policy to StoragePolicySelector.
/// Used when storage policy needs to be created on the fly, not being present in config file.
/// Done by getOrSetStoragePolicyForSingleDisk.
void add(StoragePolicyPtr storage_policy);
private:
StoragePoliciesMap policies;
};
}
|
