blob: 1b6e936274e90a821f93f6030892d2aa2a1ee4e6 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
//
// StreamTokenizer.cpp
//
// Library: Foundation
// Package: Streams
// Module: StreamTokenizer
//
// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/StreamTokenizer.h"
namespace Poco {
StreamTokenizer::StreamTokenizer():
_pIstr(0)
{
}
StreamTokenizer::StreamTokenizer(std::istream& istr):
_pIstr(&istr)
{
}
StreamTokenizer::~StreamTokenizer()
{
for (TokenVec::iterator it = _tokens.begin(); it != _tokens.end(); ++it)
{
delete it->pToken;
}
}
void StreamTokenizer::attachToStream(std::istream& istr)
{
_pIstr = &istr;
}
void StreamTokenizer::addToken(Token* pToken)
{
poco_check_ptr (pToken);
TokenInfo ti;
ti.pToken = pToken;
ti.ignore = (pToken->tokenClass() == Token::COMMENT_TOKEN || pToken->tokenClass() == Token::WHITESPACE_TOKEN);
_tokens.push_back(ti);
}
void StreamTokenizer::addToken(Token* pToken, bool ignore)
{
poco_check_ptr (pToken);
TokenInfo ti;
ti.pToken = pToken;
ti.ignore = ignore;
_tokens.push_back(ti);
}
const Token* StreamTokenizer::next()
{
poco_check_ptr (_pIstr);
static const int eof = std::char_traits<char>::eof();
int first = _pIstr->get();
TokenVec::const_iterator it = _tokens.begin();
while (first != eof && it != _tokens.end())
{
const TokenInfo& ti = *it;
if (ti.pToken->start((char) first, *_pIstr))
{
ti.pToken->finish(*_pIstr);
if (ti.ignore)
{
first = _pIstr->get();
it = _tokens.begin();
}
else return ti.pToken;
}
else ++it;
}
if (first == eof)
{
return &_eofToken;
}
else
{
_invalidToken.start((char) first, *_pIstr);
return &_invalidToken;
}
}
} // namespace Poco
|
