1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#include "markdown.h"
#include <yql/essentials/utils/yql_panic.h>
#include <contrib/libs/re2/re2/re2.h>
#include <util/generic/yexception.h>
#include <util/charset/utf8.h>
namespace NYql::NDocs {
class TMarkdownParser {
private:
static constexpr TStringBuf HeaderRegex = R"re(([^#]+)(\s+{#([a-z0-9\-_]+)})?)re";
public:
explicit TMarkdownParser(size_t headerDepth)
: HeaderDepth_(headerDepth)
, SectionHeaderRegex_(" *" + TString(HeaderDepth_, '#') + " " + HeaderRegex)
, IsSkipping_(true)
{
}
void Parse(IInputStream& markdown, TMarkdownCallback&& onSection) {
for (TString line; markdown.ReadLine(line) != 0;) {
if (IsSkipping_) {
if (IsSectionHeader(line)) {
ResetSection(std::move(line));
IsSkipping_ = false;
} else {
// Skip
}
} else {
if (IsSectionHeader(line)) {
onSection(std::move(Section_));
ResetSection(std::move(line));
} else {
line.append('\n');
Section_.Body.append(std::move(line));
}
}
}
if (!IsSkipping_) {
onSection(std::move(Section_));
}
}
private:
void ResetSection(TString&& line) {
Section_ = TMarkdownSection();
TString content;
std::optional<TString> dummy;
std::optional<TString> anchor;
if (!RE2::FullMatch(line, SectionHeaderRegex_, &content, &dummy, &anchor)) {
Section_.Header.Content = std::move(line);
return;
}
Section_.Header.Content = std::move(content);
if (anchor) {
Section_.Header.Anchor = std::move(*anchor);
}
}
bool IsSectionHeader(TStringBuf line) const {
return HeaderDepth(line) == HeaderDepth_;
}
size_t HeaderDepth(TStringBuf line) const {
size_t begin = line.find('#');
size_t end = line.find_first_not_of('#', begin);
return end != TStringBuf::npos ? (end - begin) : 0;
}
size_t HeaderDepth_;
RE2 SectionHeaderRegex_;
bool IsSkipping_;
TMarkdownSection Section_;
};
TMaybe<TString> Anchor(const TMarkdownHeader& header) {
static RE2 Regex(R"re([0-9a-z\-_]+)re");
if (header.Anchor) {
return header.Anchor;
}
TString content = ToLowerUTF8(header.Content);
SubstGlobal(content, ' ', '-');
if (RE2::FullMatch(content, Regex)) {
return content;
}
return Nothing();
}
TMarkdownPage ParseMarkdownPage(TString markdown) {
TMarkdownPage page;
const auto onSection = [&](TMarkdownSection&& section) {
if (TMaybe<TString> anchor = Anchor(section.Header)) {
section.Header.Anchor = anchor;
page.SectionsByAnchor[*anchor] = std::move(section);
}
};
{
TMarkdownParser parser(/*headerDepth=*/2);
TStringStream stream(markdown);
parser.Parse(stream, onSection);
}
{
TMarkdownParser parser(/*headerDepth=*/3);
TStringStream stream(markdown);
parser.Parse(stream, onSection);
}
page.Text = std::move(markdown);
return page;
}
} // namespace NYql::NDocs
|
