blob: 3c5b6964100320a4cd8855481789d24590cbe93b (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
//
// StreamTokenizer.h
//
// Library: Foundation
// Package: Streams
// Module: StreamTokenizer
//
// Definition of the StreamTokenizer class.
//
// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef Foundation_StreamTokenizer_INCLUDED
#define Foundation_StreamTokenizer_INCLUDED
#include "Poco/Foundation.h"
#include "Poco/Token.h"
#include <istream>
#include <vector>
namespace Poco {
class Foundation_API StreamTokenizer
/// A stream tokenizer splits an input stream
/// into a sequence of tokens of different kinds.
/// Various token kinds can be registered with
/// the tokenizer.
{
public:
StreamTokenizer();
/// Creates a StreamTokenizer with no attached stream.
StreamTokenizer(std::istream& istr);
/// Creates a StreamTokenizer with no attached stream.
virtual ~StreamTokenizer();
/// Destroys the StreamTokenizer and deletes all
/// registered tokens.
void attachToStream(std::istream& istr);
/// Attaches the tokenizer to an input stream.
void addToken(Token* pToken);
/// Adds a token class to the tokenizer. The
/// tokenizer takes ownership of the token and
/// deletes it when no longer needed. Comment
/// and whitespace tokens will be marked as
/// ignorable, which means that next() will not
/// return them.
void addToken(Token* pToken, bool ignore);
/// Adds a token class to the tokenizer. The
/// tokenizer takes ownership of the token and
/// deletes it when no longer needed.
/// If ignore is true, the token will be marked
/// as ignorable, which means that next() will
/// not return it.
const Token* next();
/// Extracts the next token from the input stream.
/// Returns a pointer to an EOFToken if there are
/// no more characters to read.
/// Returns a pointer to an InvalidToken if an
/// invalid character is encountered.
/// If a token is marked as ignorable, it will not
/// be returned, and the next token will be
/// examined.
/// Never returns a NULL pointer.
/// You must not delete the token returned by next().
private:
struct TokenInfo
{
Token* pToken;
bool ignore;
};
typedef std::vector<TokenInfo> TokenVec;
TokenVec _tokens;
std::istream* _pIstr;
InvalidToken _invalidToken;
EOFToken _eofToken;
};
} // namespace Poco
#endif // Foundation_StreamTokenizer_INCLUDED
|
