1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
#pragma once
#include <IO/ReadBufferFromFileBase.h>
#include <Interpreters/Context_fwd.h>
#include <Common/Throttler_fwd.h>
#include <unistd.h>
namespace DB
{
/** Use ready file descriptor. Does not open or close a file.
*/
class ReadBufferFromFileDescriptor : public ReadBufferFromFileBase
{
protected:
const size_t required_alignment = 0; /// For O_DIRECT both file offsets and memory addresses have to be aligned.
bool use_pread = false; /// To access one fd from multiple threads, use 'pread' syscall instead of 'read'.
size_t file_offset_of_buffer_end = 0; /// What offset in file corresponds to working_buffer.end().
int fd;
ThrottlerPtr throttler;
bool nextImpl() override;
void prefetch(Priority priority) override;
/// Name or some description of file.
std::string getFileName() const override;
/// Does the read()/pread(), with all the metric increments, error handling, throttling, etc.
/// Doesn't seek (`offset` must match fd's position if !use_pread).
/// Stops after min_bytes or eof. Returns 0 if eof.
/// Thread safe.
size_t readImpl(char * to, size_t min_bytes, size_t max_bytes, size_t offset);
public:
explicit ReadBufferFromFileDescriptor(
int fd_,
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr,
size_t alignment = 0,
std::optional<size_t> file_size_ = std::nullopt,
ThrottlerPtr throttler_ = {})
: ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_)
, required_alignment(alignment)
, fd(fd_)
, throttler(throttler_)
{
}
int getFD() const
{
return fd;
}
off_t getPosition() override
{
return file_offset_of_buffer_end - (working_buffer.end() - pos);
}
size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; }
/// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen.
off_t seek(off_t off, int whence) override;
/// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read.
void rewind();
size_t getFileSize() override;
bool checkIfActuallySeekable() override;
size_t readBigAt(char * to, size_t n, size_t offset, const std::function<bool(size_t)> &) override;
bool supportsReadAt() override { return use_pread; }
private:
/// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
bool poll(size_t timeout_microseconds) const;
};
/** Similar to ReadBufferFromFileDescriptor but it is using 'pread' allowing multiple concurrent reads from the same fd.
*/
class ReadBufferFromFileDescriptorPRead : public ReadBufferFromFileDescriptor
{
public:
explicit ReadBufferFromFileDescriptorPRead(
int fd_,
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr,
size_t alignment = 0,
std::optional<size_t> file_size_ = std::nullopt,
ThrottlerPtr throttler_ = {})
: ReadBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment, file_size_, throttler_)
{
use_pread = true;
}
};
}
|
