#include "expression_parser.h"
#include <sstream>
#include <unordered_set>
#include <jinja2cpp/template_env.h>
namespace jinja2
{
template<typename T>
auto ReplaceErrorIfPossible(T& result, const Token& pivotTok, ErrorCode newError)
{
auto& error = result.error();
if (error.errorToken.range.startOffset == pivotTok.range.startOffset)
return MakeParseError(newError, pivotTok);
return result.get_unexpected();
}
ExpressionParser::ExpressionParser(const Settings& /* settings */, TemplateEnv* /* env */)
{
}
ExpressionParser::ParseResult<RendererPtr> ExpressionParser::Parse(LexScanner& lexer)
{
auto evaluator = ParseFullExpression(lexer);
if (!evaluator)
return evaluator.get_unexpected();
auto tok = lexer.NextToken();
if (tok != Token::Eof)
{
auto tok1 = tok;
tok1.type = Token::Eof;
return MakeParseError(ErrorCode::ExpectedToken, tok, {tok1});
}
RendererPtr result = std::make_shared<ExpressionRenderer>(*evaluator);
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<FullExpressionEvaluator>> ExpressionParser::ParseFullExpression(LexScanner &lexer, bool includeIfPart)
{
ExpressionEvaluatorPtr<FullExpressionEvaluator> result;
LexScanner::StateSaver saver(lexer);
ExpressionEvaluatorPtr<FullExpressionEvaluator> evaluator = std::make_shared<FullExpressionEvaluator>();
auto value = ParseLogicalOr(lexer);
if (!value)
return value.get_unexpected();
evaluator->SetExpression(*value);
if (includeIfPart && lexer.EatIfEqual(Keyword::If))
{
auto ifExpr = ParseIfExpression(lexer);
if (!ifExpr)
return ifExpr.get_unexpected();
evaluator->SetTester(*ifExpr);
}
saver.Commit();
return evaluator;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalOr(LexScanner& lexer)
{
auto left = ParseLogicalAnd(lexer);
if (left && lexer.EatIfEqual(Keyword::LogicalOr))
{
auto right = ParseLogicalOr(lexer);
if (!right)
return right.get_unexpected();
return std::make_shared<BinaryExpression>(BinaryExpression::LogicalOr, *left, *right);
}
return left;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalAnd(LexScanner& lexer)
{
auto left = ParseLogicalCompare(lexer);
if (left && lexer.EatIfEqual(Keyword::LogicalAnd))
{
auto right = ParseLogicalAnd(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(BinaryExpression::LogicalAnd, *left, *right);
}
return left;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseLogicalCompare(LexScanner& lexer)
{
auto left = ParseStringConcat(lexer);
if (!left)
return left;
auto tok = lexer.NextToken();
BinaryExpression::Operation operation;
switch (tok.type)
{
case Token::Equal:
operation = BinaryExpression::LogicalEq;
break;
case Token::NotEqual:
operation = BinaryExpression::LogicalNe;
break;
case '<':
operation = BinaryExpression::LogicalLt;
break;
case '>':
operation = BinaryExpression::LogicalGt;
break;
case Token::GreaterEqual:
operation = BinaryExpression::LogicalGe;
break;
case Token::LessEqual:
operation = BinaryExpression::LogicalLe;
break;
default:
switch (lexer.GetAsKeyword(tok))
{
case Keyword::In:
operation = BinaryExpression::In;
break;
case Keyword::Is:
{
Token nextTok = lexer.NextToken();
if (nextTok != Token::Identifier)
return MakeParseError(ErrorCode::ExpectedIdentifier, nextTok);
std::string name = AsString(nextTok.value);
ParseResult<CallParamsInfo> params;
if (lexer.EatIfEqual('('))
params = ParseCallParams(lexer);
if (!params)
return params.get_unexpected();
return std::make_shared<IsExpression>(*left, std::move(name), std::move(*params));
}
default:
lexer.ReturnToken();
return left;
}
}
auto right = ParseStringConcat(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(operation, *left, *right);
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseStringConcat(LexScanner& lexer)
{
auto left = ParseMathPow(lexer);
if (left && lexer.EatIfEqual('~'))
{
auto right = ParseLogicalAnd(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(BinaryExpression::StringConcat, *left, *right);
}
return left;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathPow(LexScanner& lexer)
{
auto left = ParseMathPlusMinus(lexer);
if (left && lexer.EatIfEqual(Token::MulMul))
{
auto right = ParseMathPow(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(BinaryExpression::Pow, *left, *right);
}
return left;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathPlusMinus(LexScanner& lexer)
{
auto left = ParseMathMulDiv(lexer);
if (!left)
return left;
auto tok = lexer.NextToken();
BinaryExpression::Operation operation;
switch (tok.type)
{
case '+':
operation = BinaryExpression::Plus;
break;
case '-':
operation = BinaryExpression::Minus;
break;
default:
lexer.ReturnToken();
return left;
}
auto right = ParseMathPlusMinus(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(operation, *left, *right);
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseMathMulDiv(LexScanner& lexer)
{
auto left = ParseUnaryPlusMinus(lexer);
if (!left)
return left;
auto tok = lexer.NextToken();
BinaryExpression::Operation operation;
switch (tok.type)
{
case '*':
operation = BinaryExpression::Mul;
break;
case '/':
operation = BinaryExpression::Div;
break;
case Token::DivDiv:
operation = BinaryExpression::DivInteger;
break;
case '%':
operation = BinaryExpression::DivReminder;
break;
default:
lexer.ReturnToken();
return left;
}
auto right = ParseMathMulDiv(lexer);
if (!right)
return right;
return std::make_shared<BinaryExpression>(operation, *left, *right);
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseUnaryPlusMinus(LexScanner& lexer)
{
const auto tok = lexer.NextToken();
const auto isUnary = tok == '+' || tok == '-' || lexer.GetAsKeyword(tok) == Keyword::LogicalNot;
if (!isUnary)
lexer.ReturnToken();
auto subExpr = ParseValueExpression(lexer);
if (!subExpr)
return subExpr;
ExpressionEvaluatorPtr<Expression> result;
if (isUnary)
result = std::make_shared<UnaryExpression>(tok == '+' ? UnaryExpression::UnaryPlus : (tok == '-' ? UnaryExpression::UnaryMinus : UnaryExpression::LogicalNot), *subExpr);
else
result = subExpr.value();
if (lexer.EatIfEqual('|'))
{
auto filter = ParseFilterExpression(lexer);
if (!filter)
return filter.get_unexpected();
result = std::make_shared<FilteredExpression>(std::move(result), *filter);
}
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseValueExpression(LexScanner& lexer)
{
Token tok = lexer.NextToken();
static const std::unordered_set<Keyword> forbiddenKw = {Keyword::Is, Keyword::In, Keyword::If, Keyword::Else};
ParseResult<ExpressionEvaluatorPtr<Expression>> valueRef;
switch (tok.type)
{
case Token::Identifier:
{
auto kwType = lexer.GetAsKeyword(tok);
if (forbiddenKw.count(kwType) != 0)
return MakeParseError(ErrorCode::UnexpectedToken, tok);
valueRef = std::make_shared<ValueRefExpression>(AsString(tok.value));
break;
}
case Token::IntegerNum:
case Token::FloatNum:
case Token::String:
return std::make_shared<ConstantExpression>(tok.value);
case Token::True:
return std::make_shared<ConstantExpression>(InternalValue(true));
case Token::False:
return std::make_shared<ConstantExpression>(InternalValue(false));
case '(':
valueRef = ParseBracedExpressionOrTuple(lexer);
break;
case '[':
valueRef = ParseTuple(lexer);
break;
case '{':
valueRef = ParseDictionary(lexer);
break;
default:
return MakeParseError(ErrorCode::UnexpectedToken, tok);
}
if (valueRef)
{
tok = lexer.PeekNextToken();
if (tok == '[' || tok == '.')
valueRef = ParseSubscript(lexer, *valueRef);
if (lexer.EatIfEqual('('))
valueRef = ParseCall(lexer, *valueRef);
}
return valueRef;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseBracedExpressionOrTuple(LexScanner& lexer)
{
ExpressionEvaluatorPtr<Expression> result;
bool isTuple = false;
std::vector<ExpressionEvaluatorPtr<Expression>> exprs;
for (;;)
{
Token pivotTok = lexer.PeekNextToken();
auto expr = ParseFullExpression(lexer);
if (!expr)
return ReplaceErrorIfPossible(expr, pivotTok, ErrorCode::ExpectedRoundBracket);
exprs.push_back(*expr);
Token tok = lexer.NextToken();
if (tok == ')')
break;
else if (tok == ',')
isTuple = true;
}
if (isTuple)
result = std::make_shared<TupleCreator>(std::move(exprs));
else
result = exprs[0];
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseDictionary(LexScanner& lexer)
{
ExpressionEvaluatorPtr<Expression> result;
std::unordered_map<std::string, ExpressionEvaluatorPtr<Expression>> items;
if (lexer.EatIfEqual(']'))
return std::make_shared<DictCreator>(std::move(items));
do
{
Token key = lexer.NextToken();
if (key != Token::String)
return MakeParseError(ErrorCode::ExpectedStringLiteral, key);
if (!lexer.EatIfEqual('='))
{
auto tok = lexer.PeekNextToken();
auto tok1 = tok;
tok1.type = Token::Assign;
return MakeParseError(ErrorCode::ExpectedToken, tok, {tok1});
}
auto pivotTok = lexer.PeekNextToken();
auto expr = ParseFullExpression(lexer);
if (!expr)
return ReplaceErrorIfPossible(expr, pivotTok, ErrorCode::ExpectedExpression);
items[AsString(key.value)] = *expr;
} while (lexer.EatIfEqual(','));
auto tok = lexer.NextToken();
if (tok != '}')
return MakeParseError(ErrorCode::ExpectedCurlyBracket, tok);
result = std::make_shared<DictCreator>(std::move(items));
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseTuple(LexScanner& lexer)
{
ExpressionEvaluatorPtr<Expression> result;
std::vector<ExpressionEvaluatorPtr<Expression>> exprs;
if (lexer.EatIfEqual(']'))
return std::make_shared<TupleCreator>(exprs);
do
{
auto expr = ParseFullExpression(lexer);
if (!expr)
return expr.get_unexpected();
exprs.push_back(*expr);
} while (lexer.EatIfEqual(','));
auto tok = lexer.NextToken();
if (tok != ']')
return MakeParseError(ErrorCode::ExpectedSquareBracket, tok);
result = std::make_shared<TupleCreator>(std::move(exprs));
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseCall(LexScanner& lexer, ExpressionEvaluatorPtr<Expression> valueRef)
{
ExpressionEvaluatorPtr<Expression> result;
ParseResult<CallParamsInfo> params = ParseCallParams(lexer);
if (!params)
return params.get_unexpected();
result = std::make_shared<CallExpression>(valueRef, std::move(*params));
return result;
}
ExpressionParser::ParseResult<CallParamsInfo> ExpressionParser::ParseCallParams(LexScanner& lexer)
{
CallParamsInfo result;
if (lexer.EatIfEqual(')'))
return result;
do
{
Token tok = lexer.NextToken();
std::string paramName;
if (tok == Token::Identifier && lexer.PeekNextToken() == '=')
{
paramName = AsString(tok.value);
lexer.EatToken();
}
else
{
lexer.ReturnToken();
}
auto valueExpr = ParseFullExpression(lexer);
if (!valueExpr)
{
return valueExpr.get_unexpected();
}
if (paramName.empty())
result.posParams.push_back(*valueExpr);
else
result.kwParams[paramName] = *valueExpr;
} while (lexer.EatIfEqual(','));
auto tok = lexer.NextToken();
if (tok != ')')
return MakeParseError(ErrorCode::ExpectedRoundBracket, tok);
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<Expression>> ExpressionParser::ParseSubscript(LexScanner& lexer, ExpressionEvaluatorPtr<Expression> valueRef)
{
ExpressionEvaluatorPtr<SubscriptExpression> result = std::make_shared<SubscriptExpression>(valueRef);
for (Token tok = lexer.NextToken(); tok.type == '.' || tok.type == '['; tok = lexer.NextToken())
{
ParseResult<ExpressionEvaluatorPtr<Expression>> indexExpr;
if (tok == '.')
{
tok = lexer.NextToken();
if (tok.type != Token::Identifier)
return MakeParseError(ErrorCode::ExpectedIdentifier, tok);
auto valueName = AsString(tok.value);
indexExpr = std::make_shared<ConstantExpression>(InternalValue(valueName));
}
else
{
auto expr = ParseFullExpression(lexer);
if (!expr)
return expr.get_unexpected();
else
indexExpr = *expr;
if (!lexer.EatIfEqual(']', &tok))
return MakeParseError(ErrorCode::ExpectedSquareBracket, lexer.PeekNextToken());
}
result->AddIndex(*indexExpr);
}
lexer.ReturnToken();
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<ExpressionFilter>> ExpressionParser::ParseFilterExpression(LexScanner& lexer)
{
ExpressionEvaluatorPtr<ExpressionFilter> result;
auto startTok = lexer.PeekNextToken();
try
{
do
{
Token tok = lexer.NextToken();
if (tok != Token::Identifier)
return MakeParseError(ErrorCode::ExpectedIdentifier, tok);
std::string name = AsString(tok.value);
ParseResult<CallParamsInfo> params;
if (lexer.NextToken() == '(')
params = ParseCallParams(lexer);
else
lexer.ReturnToken();
if (!params)
return params.get_unexpected();
auto filter = std::make_shared<ExpressionFilter>(name, std::move(*params));
if (result)
{
filter->SetParentFilter(result);
result = filter;
}
else
result = filter;
} while (lexer.NextToken() == '|');
lexer.ReturnToken();
}
catch (const ParseError& error)
{
return nonstd::make_unexpected(error);
}
catch (const std::runtime_error&)
{
return MakeParseError(ErrorCode::UnexpectedException, startTok);
}
return result;
}
ExpressionParser::ParseResult<ExpressionEvaluatorPtr<IfExpression>> ExpressionParser::ParseIfExpression(LexScanner& lexer)
{
ExpressionEvaluatorPtr<IfExpression> result;
auto startTok = lexer.PeekNextToken();
try
{
auto testExpr = ParseLogicalOr(lexer);
if (!testExpr)
return testExpr.get_unexpected();
ParseResult<ExpressionEvaluatorPtr<>> altValue;
if (lexer.GetAsKeyword(lexer.PeekNextToken()) == Keyword::Else)
{
lexer.EatToken();
auto value = ParseFullExpression(lexer);
if (!value)
return value.get_unexpected();
altValue = *value;
}
result = std::make_shared<IfExpression>(*testExpr, *altValue);
}
catch (const ParseError& error)
{
return nonstd::make_unexpected(error);
}
catch (const std::runtime_error& ex)
{
std::cout << "Filter parsing problem: " << ex.what() << std::endl;
}
return result;
}
} // namespace jinja2