Merge branch 'rightlib' into mergelibs-241016-1210

1 files changed, 606 insertions, 0 deletions

diff --git a/contrib/libs/jinja2cpp/src/expression_parser.cpp b/contrib/libs/jinja2cpp/src/expression_parser.cpp
new file mode 100644
index 0000000000..edc599cc19
--- /dev/null
+++ b/contrib/libs/jinja2cpp/src/expression_parser.cpp
@@ -0,0 +1,606 @@
+#include "expression_parser.h"
+
+#include <sstream>
+#include <unordered_set>
+#include <jinja2cpp/template_env.h>
+
+namespace jinja2
+{
+
+template<typename T>
+auto ReplaceErrorIfPossible(T& result, const Token& pivotTok, ErrorCode newError)
+{
+    auto& error = result.error();
+    if (error.errorToken.range.startOffset == pivotTok.range.startOffset)
+        return MakeParseError(newError, pivotTok);
+
+    return result.get_unexpected();
+}
+
+ExpressionParser::ExpressionParser(const Settings& /* settings */, TemplateEnv* /* env */)
+{
+
+}
+
+ExpressionParser::ParseResult<RendererPtr> ExpressionParser::Parse(LexScanner& lexer)
+{
+    auto evaluator = ParseFullExpression(lexer);
+    if (!evaluator)
+        return evaluator.get_unexpected();
+
+    auto tok = lexer.NextToken();
+    if (tok != Token::Eof)
+    {
+        auto tok1 = tok;
+        tok1.type = Token::Eof;
+
+        return MakeParseError(ErrorCode::ExpectedToken, tok, {tok1});
+    }
+
+    RendererPtr result = std::make_shared<ExpressionRenderer>(*evaluator);
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<FullExpressionEvaluator>> ExpressionParser::ParseFullExpression(LexScanner &lexer, bool includeIfPart)
+{
+    ExpressionEvaluatorPtr<FullExpressionEvaluator> result;
+    LexScanner::StateSaver saver(lexer);
+
+    ExpressionEvaluatorPtr<FullExpressionEvaluator> evaluator = std::make_shared<FullExpressionEvaluator>();
+    auto value = ParseLogicalOr(lexer);
+    if (!value)
+        return value.get_unexpected();
+
+    evaluator->SetExpression(*value);
+
+    if (includeIfPart && lexer.EatIfEqual(Keyword::If))
+    {
+        auto ifExpr = ParseIfExpression(lexer);
+        if (!ifExpr)
+            return ifExpr.get_unexpected();
+        evaluator->SetTester(*ifExpr);
+    }
+
+    saver.Commit();
+
+    return evaluator;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalOr(LexScanner& lexer)
+{
+    auto left = ParseLogicalAnd(lexer);
+
+    if (left && lexer.EatIfEqual(Keyword::LogicalOr))
+    {
+        auto right = ParseLogicalOr(lexer);
+        if (!right)
+            return right.get_unexpected();
+
+        return std::make_shared<BinaryExpression>(BinaryExpression::LogicalOr, *left, *right);
+    }
+
+    return left;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalAnd(LexScanner& lexer)
+{
+    auto left = ParseLogicalCompare(lexer);
+
+    if (left && lexer.EatIfEqual(Keyword::LogicalAnd))
+    {
+        auto right = ParseLogicalAnd(lexer);
+        if (!right)
+            return right;
+
+        return std::make_shared<BinaryExpression>(BinaryExpression::LogicalAnd, *left, *right);
+    }
+
+    return left;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalCompare(LexScanner& lexer)
+{
+    auto left = ParseStringConcat(lexer);
+    if (!left)
+        return left;
+
+    auto tok = lexer.NextToken();
+    BinaryExpression::Operation operation;
+    switch (tok.type)
+    {
+    case Token::Equal:
+        operation = BinaryExpression::LogicalEq;
+        break;
+    case Token::NotEqual:
+        operation = BinaryExpression::LogicalNe;
+        break;
+    case '<':
+        operation = BinaryExpression::LogicalLt;
+        break;
+    case '>':
+        operation = BinaryExpression::LogicalGt;
+        break;
+    case Token::GreaterEqual:
+        operation = BinaryExpression::LogicalGe;
+        break;
+    case Token::LessEqual:
+        operation = BinaryExpression::LogicalLe;
+        break;
+    default:
+        switch (lexer.GetAsKeyword(tok))
+        {
+        case Keyword::In:
+            operation = BinaryExpression::In;
+            break;
+        case Keyword::Is:
+        {
+            Token nextTok = lexer.NextToken();
+            if (nextTok != Token::Identifier)
+                return MakeParseError(ErrorCode::ExpectedIdentifier, nextTok);
+    
+            std::string name = AsString(nextTok.value);
+            ParseResult<CallParamsInfo> params;
+
+            if (lexer.EatIfEqual('('))
+                params = ParseCallParams(lexer);
+    
+            if (!params)
+                return params.get_unexpected();
+    
+            return std::make_shared<IsExpression>(*left, std::move(name), std::move(*params));
+        }
+        default:
+            lexer.ReturnToken();
+            return left;            
+        }
+    }
+
+    auto right = ParseStringConcat(lexer);
+    if (!right)
+        return right;
+
+    return std::make_shared<BinaryExpression>(operation, *left, *right);
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseStringConcat(LexScanner& lexer)
+{
+    auto left = ParseMathPow(lexer);
+
+    if (left && lexer.EatIfEqual('~'))
+    {
+        auto right = ParseLogicalAnd(lexer);
+        if (!right)
+            return right;
+
+        return std::make_shared<BinaryExpression>(BinaryExpression::StringConcat, *left, *right);
+    }
+    return left;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathPow(LexScanner& lexer)
+{
+    auto left = ParseMathPlusMinus(lexer);
+
+    if (left && lexer.EatIfEqual(Token::MulMul))
+    {
+        auto right = ParseMathPow(lexer);
+        if (!right)
+            return right;
+
+        return std::make_shared<BinaryExpression>(BinaryExpression::Pow, *left, *right);
+    }
+
+    return left;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathPlusMinus(LexScanner& lexer)
+{
+    auto left = ParseMathMulDiv(lexer);
+    if (!left)
+        return left;
+
+    auto tok = lexer.NextToken();
+    BinaryExpression::Operation operation;
+    switch (tok.type)
+    {
+    case '+':
+        operation = BinaryExpression::Plus;
+        break;
+    case '-':
+        operation = BinaryExpression::Minus;
+        break;
+    default:
+        lexer.ReturnToken();
+        return left;
+    }
+
+    auto right = ParseMathPlusMinus(lexer);
+    if (!right)
+        return right;
+
+    return std::make_shared<BinaryExpression>(operation, *left, *right);
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathMulDiv(LexScanner& lexer)
+{
+    auto left = ParseUnaryPlusMinus(lexer);
+    if (!left)
+        return left;
+
+    auto tok = lexer.NextToken();
+    BinaryExpression::Operation operation;
+    switch (tok.type)
+    {
+    case '*':
+        operation = BinaryExpression::Mul;
+        break;
+    case '/':
+        operation = BinaryExpression::Div;
+        break;
+    case Token::DivDiv:
+        operation = BinaryExpression::DivInteger;
+        break;
+    case '%':
+        operation = BinaryExpression::DivReminder;
+        break;
+    default:
+        lexer.ReturnToken();
+        return left;
+    }
+
+    auto right = ParseMathMulDiv(lexer);
+    if (!right)
+        return right;
+
+    return std::make_shared<BinaryExpression>(operation, *left, *right);
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseUnaryPlusMinus(LexScanner& lexer)
+{
+    const auto tok = lexer.NextToken();
+    const auto isUnary = tok == '+' || tok == '-' || lexer.GetAsKeyword(tok) == Keyword::LogicalNot;
+    if (!isUnary)
+        lexer.ReturnToken();
+  
+    auto subExpr = ParseValueExpression(lexer);
+    if (!subExpr)
+        return subExpr;
+
+    ExpressionEvaluatorPtr<Expression> result;
+    if (isUnary)
+        result = std::make_shared<UnaryExpression>(tok == '+' ? UnaryExpression::UnaryPlus : (tok == '-' ? UnaryExpression::UnaryMinus : UnaryExpression::LogicalNot), *subExpr);
+    else
+        result = subExpr.value();
+
+    if (lexer.EatIfEqual('|'))
+    {
+        auto filter = ParseFilterExpression(lexer);
+        if (!filter)
+            return filter.get_unexpected();
+        result = std::make_shared<FilteredExpression>(std::move(result), *filter);
+    }
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseValueExpression(LexScanner& lexer)
+{
+    Token tok = lexer.NextToken();
+    static const std::unordered_set<Keyword> forbiddenKw = {Keyword::Is, Keyword::In, Keyword::If, Keyword::Else};
+
+    ParseResult<ExpressionEvaluatorPtr<Expression>> valueRef;
+
+    switch (tok.type)
+    {
+    case Token::Identifier:
+    {
+        auto kwType = lexer.GetAsKeyword(tok);
+        if (forbiddenKw.count(kwType) != 0)
+            return MakeParseError(ErrorCode::UnexpectedToken, tok);
+            
+        valueRef = std::make_shared<ValueRefExpression>(AsString(tok.value));
+        break;
+    }
+    case Token::IntegerNum:
+    case Token::FloatNum:
+    case Token::String:
+        return std::make_shared<ConstantExpression>(tok.value);
+    case Token::True:
+        return std::make_shared<ConstantExpression>(InternalValue(true));
+    case Token::False:
+        return std::make_shared<ConstantExpression>(InternalValue(false));
+    case '(':
+        valueRef = ParseBracedExpressionOrTuple(lexer);
+        break;
+    case '[':
+        valueRef = ParseTuple(lexer);
+        break;
+    case '{':
+        valueRef = ParseDictionary(lexer);
+        break;
+    default:
+        return MakeParseError(ErrorCode::UnexpectedToken, tok);
+    }
+
+    if (valueRef)
+    {
+        tok = lexer.PeekNextToken();
+        if (tok == '[' || tok == '.')
+            valueRef = ParseSubscript(lexer, *valueRef);
+
+        if (lexer.EatIfEqual('('))
+            valueRef = ParseCall(lexer, *valueRef);
+    }
+    return valueRef;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseBracedExpressionOrTuple(LexScanner& lexer)
+{
+    ExpressionEvaluatorPtr<Expression> result;
+
+    bool isTuple = false;
+    std::vector<ExpressionEvaluatorPtr<Expression>> exprs;
+    for (;;)
+    {
+        Token pivotTok = lexer.PeekNextToken();
+        auto expr = ParseFullExpression(lexer);
+
+        if (!expr)
+            return ReplaceErrorIfPossible(expr, pivotTok, ErrorCode::ExpectedRoundBracket);
+
+        exprs.push_back(*expr);
+        Token tok = lexer.NextToken();
+        if (tok == ')')
+            break;
+        else if (tok == ',')
+            isTuple = true;
+    }
+
+    if (isTuple)
+        result = std::make_shared<TupleCreator>(std::move(exprs));
+    else
+        result = exprs[0];
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseDictionary(LexScanner& lexer)
+{
+    ExpressionEvaluatorPtr<Expression> result;
+
+    std::unordered_map<std::string, ExpressionEvaluatorPtr<Expression>> items;
+    if (lexer.EatIfEqual(']'))
+        return std::make_shared<DictCreator>(std::move(items));
+
+    do
+    {
+        Token key = lexer.NextToken();
+        if (key != Token::String)
+            return MakeParseError(ErrorCode::ExpectedStringLiteral, key);
+
+        if (!lexer.EatIfEqual('='))
+        {
+            auto tok = lexer.PeekNextToken();
+            auto tok1 = tok;
+            tok1.type = Token::Assign;
+            return MakeParseError(ErrorCode::ExpectedToken, tok, {tok1});
+        }
+
+        auto pivotTok = lexer.PeekNextToken();
+        auto expr = ParseFullExpression(lexer);
+        if (!expr)
+            return ReplaceErrorIfPossible(expr, pivotTok, ErrorCode::ExpectedExpression);
+
+        items[AsString(key.value)] = *expr;
+
+    } while (lexer.EatIfEqual(','));
+
+    auto tok = lexer.NextToken();
+    if (tok != '}')
+        return MakeParseError(ErrorCode::ExpectedCurlyBracket, tok);
+
+    result = std::make_shared<DictCreator>(std::move(items));
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseTuple(LexScanner& lexer)
+{
+    ExpressionEvaluatorPtr<Expression> result;
+
+    std::vector<ExpressionEvaluatorPtr<Expression>> exprs;
+    if (lexer.EatIfEqual(']'))
+        return std::make_shared<TupleCreator>(exprs);
+
+    do
+    {
+        auto expr = ParseFullExpression(lexer);
+        if (!expr)
+            return expr.get_unexpected();
+
+        exprs.push_back(*expr);
+    } while (lexer.EatIfEqual(','));
+
+    auto tok = lexer.NextToken();
+    if (tok != ']')
+        return MakeParseError(ErrorCode::ExpectedSquareBracket, tok);
+
+    result = std::make_shared<TupleCreator>(std::move(exprs));
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseCall(LexScanner& lexer, ExpressionEvaluatorPtr<Expression> valueRef)
+{
+    ExpressionEvaluatorPtr<Expression> result;
+
+    ParseResult<CallParamsInfo> params = ParseCallParams(lexer);
+    if (!params)
+        return params.get_unexpected();
+
+    result = std::make_shared<CallExpression>(valueRef, std::move(*params));
+
+    return result;
+}
+
+ExpressionParser::ParseResult<CallParamsInfo> ExpressionParser::ParseCallParams(LexScanner& lexer)
+{
+    CallParamsInfo result;
+
+    if (lexer.EatIfEqual(')'))
+        return result;
+
+    do
+    {
+        Token tok = lexer.NextToken();
+        std::string paramName;
+        if (tok == Token::Identifier && lexer.PeekNextToken() == '=')
+        {
+            paramName = AsString(tok.value);
+            lexer.EatToken();
+        }
+        else
+        {
+            lexer.ReturnToken();
+        }
+
+        auto valueExpr = ParseFullExpression(lexer);
+        if (!valueExpr)
+        {
+            return valueExpr.get_unexpected();
+        }
+        if (paramName.empty())
+            result.posParams.push_back(*valueExpr);
+        else
+            result.kwParams[paramName] = *valueExpr;
+
+    } while (lexer.EatIfEqual(','));
+
+    auto tok = lexer.NextToken();
+    if (tok != ')')
+        return MakeParseError(ErrorCode::ExpectedRoundBracket, tok);
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseSubscript(LexScanner& lexer, ExpressionEvaluatorPtr<Expression> valueRef)
+{
+    ExpressionEvaluatorPtr<SubscriptExpression> result = std::make_shared<SubscriptExpression>(valueRef);
+    for (Token tok = lexer.NextToken(); tok.type == '.' || tok.type == '['; tok = lexer.NextToken())
+    {
+        ParseResult<ExpressionEvaluatorPtr<Expression>> indexExpr;
+        if (tok == '.')
+        {
+            tok = lexer.NextToken();
+            if (tok.type != Token::Identifier)
+                return MakeParseError(ErrorCode::ExpectedIdentifier, tok);
+
+            auto valueName = AsString(tok.value);
+            indexExpr = std::make_shared<ConstantExpression>(InternalValue(valueName));
+        }
+        else
+        {
+            auto expr = ParseFullExpression(lexer);
+
+            if (!expr)
+                return expr.get_unexpected();
+            else
+                indexExpr = *expr;
+
+            if (!lexer.EatIfEqual(']', &tok))
+                return MakeParseError(ErrorCode::ExpectedSquareBracket, lexer.PeekNextToken());
+        }
+
+        result->AddIndex(*indexExpr);
+    }
+
+    lexer.ReturnToken();
+
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<ExpressionFilter>> ExpressionParser::ParseFilterExpression(LexScanner& lexer)
+{
+    ExpressionEvaluatorPtr<ExpressionFilter> result;
+
+    auto startTok = lexer.PeekNextToken();
+    try
+    {
+        do
+        {
+            Token tok = lexer.NextToken();
+            if (tok != Token::Identifier)
+                return MakeParseError(ErrorCode::ExpectedIdentifier, tok);
+
+            std::string name = AsString(tok.value);
+            ParseResult<CallParamsInfo> params;
+
+            if (lexer.NextToken() == '(')
+                params = ParseCallParams(lexer);
+            else
+                lexer.ReturnToken();
+
+            if (!params)
+                return params.get_unexpected();
+
+            auto filter = std::make_shared<ExpressionFilter>(name, std::move(*params));
+            if (result)
+            {
+                filter->SetParentFilter(result);
+                result = filter;
+            }
+            else
+                result = filter;
+
+        } while (lexer.NextToken() == '|');
+
+        lexer.ReturnToken();
+    }
+    catch (const ParseError& error)
+    {
+        return nonstd::make_unexpected(error);
+    }
+    catch (const std::runtime_error&)
+    {
+        return MakeParseError(ErrorCode::UnexpectedException, startTok);
+    }
+    return result;
+}
+
+ExpressionParser::ParseResult<ExpressionEvaluatorPtr<IfExpression>> ExpressionParser::ParseIfExpression(LexScanner& lexer)
+{
+    ExpressionEvaluatorPtr<IfExpression> result;
+
+    auto startTok = lexer.PeekNextToken();
+    try
+    {
+        auto testExpr = ParseLogicalOr(lexer);
+        if (!testExpr)
+            return testExpr.get_unexpected();
+
+        ParseResult<ExpressionEvaluatorPtr<>> altValue;
+        if (lexer.GetAsKeyword(lexer.PeekNextToken()) == Keyword::Else)
+        {
+            lexer.EatToken();
+            auto value = ParseFullExpression(lexer);
+            if (!value)
+                return value.get_unexpected();
+            altValue = *value;
+        }
+
+        result = std::make_shared<IfExpression>(*testExpr, *altValue);
+    }
+    catch (const ParseError& error)
+    {
+        return nonstd::make_unexpected(error);
+    }
+    catch (const std::runtime_error& ex)
+    {
+        std::cout << "Filter parsing problem: " << ex.what() << std::endl;
+    }
+
+    return result;
+}
+
+} // namespace jinja2