1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
#pragma once
#include "clickhouse_config.h"
#if USE_ORC
#include <IO/WriteBuffer.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Formats/FormatSettings.h>
#error #include <orc/OrcFile.hh>
namespace DB
{
class WriteBuffer;
/// orc::Writer writes only in orc::OutputStream
class ORCOutputStream : public orc::OutputStream
{
public:
explicit ORCOutputStream(WriteBuffer & out_);
uint64_t getLength() const override;
uint64_t getNaturalWriteSize() const override;
void write(const void * buf, size_t length) override;
void close() override {}
const std::string& getName() const override { return name; }
private:
WriteBuffer & out;
std::string name = "ORCOutputStream";
};
class ORCBlockOutputFormat : public IOutputFormat
{
public:
ORCBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_);
String getName() const override { return "ORCBlockOutputFormat"; }
private:
void consume(Chunk chunk) override;
void finalizeImpl() override;
void resetFormatterImpl() override;
std::unique_ptr<orc::Type> getORCType(const DataTypePtr & type);
/// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be
/// converted to unsigned char (bugprone-signed-char-misuse in clang).
template <typename NumberType, typename NumberVectorBatch, typename ConvertFunc>
void writeNumbers(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray<UInt8> * null_bytemap, ConvertFunc convert);
/// ConvertFunc is needed to convert ClickHouse Int128 to ORC Int128.
template <typename Decimal, typename DecimalVectorBatch, typename ConvertFunc>
void writeDecimals(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type,
const PaddedPODArray<UInt8> * null_bytemap, ConvertFunc convert);
template <typename ColumnType>
void writeStrings(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray<UInt8> * null_bytemap);
/// ORC column TimestampVectorBatch stores only seconds and nanoseconds,
/// GetSecondsFunc and GetNanosecondsFunc are needed to extract them from DataTime type.
template <typename ColumnType, typename GetSecondsFunc, typename GetNanosecondsFunc>
void writeDateTimes(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray<UInt8> * null_bytemap,
GetSecondsFunc get_seconds, GetNanosecondsFunc get_nanoseconds);
void writeColumn(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type, const PaddedPODArray<UInt8> * null_bytemap);
/// These two functions are needed to know maximum nested size of arrays to
/// create an ORC Batch with the appropriate size
size_t getColumnSize(const IColumn & column, DataTypePtr & type);
size_t getMaxColumnSize(Chunk & chunk);
void prepareWriter();
const FormatSettings format_settings;
ORCOutputStream output_stream;
DataTypes data_types;
std::unique_ptr<orc::Writer> writer;
std::unique_ptr<orc::Type> schema;
orc::WriterOptions options;
};
}
#endif
|
