library/cpp/robots_txt/robots_txt_parser.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

#pragma once

#include <algorithm>
#include <util/generic/string.h>
#include <util/generic/vector.h>
#include <util/stream/input.h>

class TRobotsTxtParser;

class TRobotsTxtRulesRecord {
private:
    TRobotsTxtParser& Parser;

public:
    TRobotsTxtRulesRecord(TRobotsTxtParser& parser);
    bool NextPair(TString& field, TString& value, bool handleErrors, TVector<int>& nonRobotsLines, bool* wasBlank = nullptr);
};

class TRobotsTxtParser {
    friend class TRobotsTxtRulesRecord;

private:
    IInputStream& InputStream;
    TString Line;
    int LineNumber;
    bool IsLastSymbolCR;

    const char* ReadLine();
    static bool IsBlankLine(const char*);
    static bool IsRobotsLine(const char*);

public:
    static char* Trim(char*);
    TRobotsTxtParser(IInputStream& inputStream);
    bool HasRecord();
    TRobotsTxtRulesRecord NextRecord();
    int GetLineNumber();
};