diff options
author | robot-piglet <[email protected]> | 2023-12-01 16:59:11 +0300 |
---|---|---|
committer | robot-piglet <[email protected]> | 2023-12-01 19:54:31 +0300 |
commit | 3715aa9254f65ae1058290101351a72a6d3a67d4 (patch) | |
tree | 9ac5a1cdab42dfc7cd095a06a362e0681cb1482f /contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp | |
parent | b20a8c04fb7e595955ca9d1b943033342b6580cb (diff) |
Intermediate changes
Diffstat (limited to 'contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp')
-rw-r--r-- | contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp | 270 |
1 files changed, 0 insertions, 270 deletions
diff --git a/contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp b/contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp deleted file mode 100644 index 16ff49e3328..00000000000 --- a/contrib/libs/antlr4_cpp_runtime/src/UnbufferedTokenStream.cpp +++ /dev/null @@ -1,270 +0,0 @@ -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - * Use of this file is governed by the BSD 3-clause license that - * can be found in the LICENSE.txt file in the project root. - */ - -#include "Token.h" -#include "Exceptions.h" -#include "assert.h" -#include "TokenSource.h" -#include "support/Arrays.h" -#include "misc/Interval.h" -#include "RuleContext.h" -#include "WritableToken.h" - -#include "UnbufferedTokenStream.h" - -using namespace antlr4; - -UnbufferedTokenStream::UnbufferedTokenStream(TokenSource *tokenSource) : UnbufferedTokenStream(tokenSource, 256) { -} - -UnbufferedTokenStream::UnbufferedTokenStream(TokenSource *tokenSource, int /*bufferSize*/) - : _tokenSource(tokenSource), _lastToken(nullptr), _lastTokenBufferStart(nullptr) -{ - InitializeInstanceFields(); - fill(1); // prime the pump -} - -UnbufferedTokenStream::~UnbufferedTokenStream() { -} - -Token* UnbufferedTokenStream::get(size_t i) const -{ // get absolute index - size_t bufferStartIndex = getBufferStartIndex(); - if (i < bufferStartIndex || i >= bufferStartIndex + _tokens.size()) { - throw IndexOutOfBoundsException(std::string("get(") + std::to_string(i) + std::string(") outside buffer: ") - + std::to_string(bufferStartIndex) + std::string("..") + std::to_string(bufferStartIndex + _tokens.size())); - } - return _tokens[i - bufferStartIndex].get(); -} - -Token* UnbufferedTokenStream::LT(ssize_t i) -{ - if (i == -1) { - return _lastToken; - } - - sync(i); - ssize_t index = static_cast<ssize_t>(_p) + i - 1; - if (index < 0) { - throw IndexOutOfBoundsException(std::string("LT(") + std::to_string(i) + std::string(") gives negative index")); - } - - if (index >= static_cast<ssize_t>(_tokens.size())) { - assert(_tokens.size() > 0 && _tokens.back()->getType() == EOF); - return _tokens.back().get(); - } - - return _tokens[static_cast<size_t>(index)].get(); -} - -size_t UnbufferedTokenStream::LA(ssize_t i) -{ - return LT(i)->getType(); -} - -TokenSource* UnbufferedTokenStream::getTokenSource() const -{ - return _tokenSource; -} - -std::string UnbufferedTokenStream::getText() -{ - return ""; -} - -std::string UnbufferedTokenStream::getText(RuleContext* ctx) -{ - return getText(ctx->getSourceInterval()); -} - -std::string UnbufferedTokenStream::getText(Token *start, Token *stop) -{ - return getText(misc::Interval(start->getTokenIndex(), stop->getTokenIndex())); -} - -void UnbufferedTokenStream::consume() -{ - if (LA(1) == EOF) { - throw IllegalStateException("cannot consume EOF"); - } - - // buf always has at least tokens[p==0] in this method due to ctor - _lastToken = _tokens[_p].get(); // track last token for LT(-1) - - // if we're at last token and no markers, opportunity to flush buffer - if (_p == _tokens.size() - 1 && _numMarkers == 0) { - _tokens.clear(); - _p = 0; - _lastTokenBufferStart = _lastToken; - } else { - ++_p; - } - - ++_currentTokenIndex; - sync(1); -} - -/// <summary> -/// Make sure we have 'need' elements from current position <seealso cref="#p p"/>. Last valid -/// {@code p} index is {@code tokens.length-1}. {@code p+need-1} is the tokens index 'need' elements -/// ahead. If we need 1 element, {@code (p+1-1)==p} must be less than {@code tokens.length}. -/// </summary> -void UnbufferedTokenStream::sync(ssize_t want) -{ - ssize_t need = (static_cast<ssize_t>(_p) + want - 1) - static_cast<ssize_t>(_tokens.size()) + 1; // how many more elements we need? - if (need > 0) { - fill(static_cast<size_t>(need)); - } -} - -/// <summary> -/// Add {@code n} elements to the buffer. Returns the number of tokens -/// actually added to the buffer. If the return value is less than {@code n}, -/// then EOF was reached before {@code n} tokens could be added. -/// </summary> -size_t UnbufferedTokenStream::fill(size_t n) -{ - for (size_t i = 0; i < n; i++) { - if (_tokens.size() > 0 && _tokens.back()->getType() == EOF) { - return i; - } - - add(_tokenSource->nextToken()); - } - - return n; -} - -void UnbufferedTokenStream::add(std::unique_ptr<Token> t) -{ - WritableToken *writable = dynamic_cast<WritableToken *>(t.get()); - if (writable != nullptr) { - writable->setTokenIndex(int(getBufferStartIndex() + _tokens.size())); - } - - _tokens.push_back(std::move(t)); -} - -/// <summary> -/// Return a marker that we can release later. -/// <p/> -/// The specific marker value used for this class allows for some level of -/// protection against misuse where {@code seek()} is called on a mark or -/// {@code release()} is called in the wrong order. -/// </summary> -ssize_t UnbufferedTokenStream::mark() -{ - if (_numMarkers == 0) { - _lastTokenBufferStart = _lastToken; - } - - int mark = -_numMarkers - 1; - _numMarkers++; - return mark; -} - -void UnbufferedTokenStream::release(ssize_t marker) -{ - ssize_t expectedMark = -_numMarkers; - if (marker != expectedMark) { - throw IllegalStateException("release() called with an invalid marker."); - } - - _numMarkers--; - if (_numMarkers == 0) { // can we release buffer? - if (_p > 0) { - // Copy tokens[p]..tokens[n-1] to tokens[0]..tokens[(n-1)-p], reset ptrs - // p is last valid token; move nothing if p==n as we have no valid char - _tokens.erase(_tokens.begin(), _tokens.begin() + static_cast<ssize_t>(_p)); - _p = 0; - } - - _lastTokenBufferStart = _lastToken; - } -} - -size_t UnbufferedTokenStream::index() -{ - return _currentTokenIndex; -} - -void UnbufferedTokenStream::seek(size_t index) -{ // seek to absolute index - if (index == _currentTokenIndex) { - return; - } - - if (index > _currentTokenIndex) { - sync(ssize_t(index - _currentTokenIndex)); - index = std::min(index, getBufferStartIndex() + _tokens.size() - 1); - } - - size_t bufferStartIndex = getBufferStartIndex(); - if (bufferStartIndex > index) { - throw IllegalArgumentException(std::string("cannot seek to negative index ") + std::to_string(index)); - } - - size_t i = index - bufferStartIndex; - if (i >= _tokens.size()) { - throw UnsupportedOperationException(std::string("seek to index outside buffer: ") + std::to_string(index) + - " not in " + std::to_string(bufferStartIndex) + ".." + std::to_string(bufferStartIndex + _tokens.size())); - } - - _p = i; - _currentTokenIndex = index; - if (_p == 0) { - _lastToken = _lastTokenBufferStart; - } else { - _lastToken = _tokens[_p - 1].get(); - } -} - -size_t UnbufferedTokenStream::size() -{ - throw UnsupportedOperationException("Unbuffered stream cannot know its size"); -} - -std::string UnbufferedTokenStream::getSourceName() const -{ - return _tokenSource->getSourceName(); -} - -std::string UnbufferedTokenStream::getText(const misc::Interval &interval) -{ - size_t bufferStartIndex = getBufferStartIndex(); - size_t bufferStopIndex = bufferStartIndex + _tokens.size() - 1; - - size_t start = interval.a; - size_t stop = interval.b; - if (start < bufferStartIndex || stop > bufferStopIndex) { - throw UnsupportedOperationException(std::string("interval ") + interval.toString() + - " not in token buffer window: " + std::to_string(bufferStartIndex) + ".." + std::to_string(bufferStopIndex)); - } - - size_t a = start - bufferStartIndex; - size_t b = stop - bufferStartIndex; - - std::stringstream ss; - for (size_t i = a; i <= b; i++) { - Token *t = _tokens[i].get(); - if (i > 0) - ss << ", "; - ss << t->getText(); - } - - return ss.str(); -} - -size_t UnbufferedTokenStream::getBufferStartIndex() const -{ - return _currentTokenIndex - _p; -} - -void UnbufferedTokenStream::InitializeInstanceFields() -{ - _p = 0; - _numMarkers = 0; - _currentTokenIndex = 0; -} |