1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
#pragma once
#include <Core/Block.h>
#include <Formats/FormatSettings.h>
#include <Processors/Formats/RowInputFormatWithNamesAndTypes.h>
#include <Processors/Formats/ISchemaReader.h>
namespace DB
{
/** A stream to input data in tsv format.
*/
class TabSeparatedRowInputFormat final : public RowInputFormatWithNamesAndTypes
{
public:
/** with_names - the first line is the header with the names of the columns
* with_types - on the next line header with type names
*/
TabSeparatedRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_,
bool with_names_, bool with_types_, bool is_raw, const FormatSettings & format_settings_);
String getName() const override { return "TabSeparatedRowInputFormat"; }
void setReadBuffer(ReadBuffer & in_) override;
void resetParser() override;
private:
TabSeparatedRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> in_, const Params & params_,
bool with_names_, bool with_types_, bool is_raw, const FormatSettings & format_settings_);
bool allowSyncAfterError() const override { return true; }
void syncAfterError() override;
bool isGarbageAfterField(size_t, ReadBuffer::Position pos) override { return *pos != '\n' && *pos != '\t'; }
bool supportsCountRows() const override { return true; }
std::unique_ptr<PeekableReadBuffer> buf;
};
class TabSeparatedFormatReader final : public FormatWithNamesAndTypesReader
{
public:
TabSeparatedFormatReader(PeekableReadBuffer & in_, const FormatSettings & format_settings, bool is_raw_);
bool readField(IColumn & column, const DataTypePtr & type,
const SerializationPtr & serialization, bool is_last_file_column, const String & column_name) override;
void skipField(size_t /*file_column*/) override { skipField(); }
void skipField();
void skipHeaderRow();
void skipNames() override { skipHeaderRow(); }
void skipTypes() override { skipHeaderRow(); }
void skipFieldDelimiter() override;
void skipRowEndDelimiter() override;
void skipPrefixBeforeHeader() override;
std::vector<String> readRow() { return readRowImpl<false>(); }
std::vector<String> readNames() override { return readHeaderRow(); }
std::vector<String> readTypes() override { return readHeaderRow(); }
std::vector<String> readHeaderRow() { return readRowImpl<true>(); }
void skipRow() override;
template <bool read_string>
String readFieldIntoString();
std::vector<String> readRowForHeaderDetection() override { return readHeaderRow(); }
void checkNullValueForNonNullable(DataTypePtr type) override;
bool parseFieldDelimiterWithDiagnosticInfo(WriteBuffer & out) override;
bool parseRowEndWithDiagnosticInfo(WriteBuffer & out) override;
FormatSettings::EscapingRule getEscapingRule() const override
{
return is_raw ? FormatSettings::EscapingRule::Raw : FormatSettings::EscapingRule::Escaped;
}
void setReadBuffer(ReadBuffer & in_) override;
bool checkForSuffix() override;
bool checkForEndOfRow() override;
bool allowVariableNumberOfColumns() const override { return format_settings.tsv.allow_variable_number_of_columns; }
private:
template <bool is_header>
std::vector<String> readRowImpl();
PeekableReadBuffer * buf;
bool is_raw;
bool first_row = true;
};
class TabSeparatedSchemaReader : public FormatWithNamesAndTypesSchemaReader
{
public:
TabSeparatedSchemaReader(ReadBuffer & in_, bool with_names_, bool with_types_, bool is_raw_, const FormatSettings & format_settings);
private:
bool allowVariableNumberOfColumns() const override { return format_settings.tsv.allow_variable_number_of_columns; }
std::optional<DataTypes> readRowAndGetDataTypesImpl() override;
std::optional<std::pair<std::vector<String>, DataTypes>> readRowAndGetFieldsAndDataTypes() override;
PeekableReadBuffer buf;
TabSeparatedFormatReader reader;
};
}
|
