Simplify CXFA_FMToken creation This CL converts the CXFA_FMToken usages into an object instead of a pointer. A copy constructor has been added. The line number was removed from the token and is retrieved from the lexer where needed. Change-Id: I94c632653e9bf1439d2ddf374a816ae0d10b5b67 Reviewed-on: https://pdfium-review.googlesource.com/27192 Commit-Queue: Ryan Harrison <rharrison@chromium.org> Reviewed-by: Ryan Harrison <rharrison@chromium.org>
diff --git a/xfa/fxfa/fm2js/cxfa_fmlexer.cpp b/xfa/fxfa/fm2js/cxfa_fmlexer.cpp index 5ae1225..78e99a5 100644 --- a/xfa/fxfa/fm2js/cxfa_fmlexer.cpp +++ b/xfa/fxfa/fm2js/cxfa_fmlexer.cpp
@@ -110,12 +110,13 @@ } // namespace -CXFA_FMToken::CXFA_FMToken() : m_type(TOKreserver), m_line_num(1) {} +CXFA_FMToken::CXFA_FMToken(XFA_FM_TOKEN token) : m_type(token) {} -CXFA_FMToken::CXFA_FMToken(uint32_t line_num) - : m_type(TOKreserver), m_line_num(line_num) {} +CXFA_FMToken::CXFA_FMToken() : CXFA_FMToken(TOKreserver) {} -CXFA_FMToken::~CXFA_FMToken() {} +CXFA_FMToken::CXFA_FMToken(const CXFA_FMToken&) = default; + +CXFA_FMToken::~CXFA_FMToken() = default; #ifndef NDEBUG WideString CXFA_FMToken::ToDebugString() const { @@ -123,8 +124,6 @@ str += tokenStrings[m_type]; str += L", string = "; str += m_string; - str += L", line_num = "; - str += std::to_wstring(m_line_num).c_str(); return str; } #endif // NDEBUG @@ -137,21 +136,19 @@ CXFA_FMLexer::~CXFA_FMLexer() {} -std::unique_ptr<CXFA_FMToken> CXFA_FMLexer::NextToken() { +CXFA_FMToken CXFA_FMLexer::NextToken() { if (m_lexer_error) - return nullptr; + return CXFA_FMToken(); - m_token = pdfium::MakeUnique<CXFA_FMToken>(m_current_line); while (m_cursor <= m_end && *m_cursor) { if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } switch (*m_cursor) { case '\n': ++m_current_line; - m_token->m_line_num = m_current_line; ++m_cursor; break; case '\r': @@ -161,9 +158,7 @@ AdvanceForComment(); break; case '"': - m_token->m_type = TOKstring; - AdvanceForString(); - return std::move(m_token); + return AdvanceForString(); case '0': case '1': case '2': @@ -174,153 +169,125 @@ case '7': case '8': case '9': - m_token->m_type = TOKnumber; - AdvanceForNumber(); - return std::move(m_token); + return AdvanceForNumber(); case '=': ++m_cursor; - if (m_cursor > m_end) { - m_token->m_type = TOKassign; - return std::move(m_token); - } + if (m_cursor > m_end) + return CXFA_FMToken(TOKassign); if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } if (*m_cursor == '=') { - m_token->m_type = TOKeq; ++m_cursor; - } else { - m_token->m_type = TOKassign; + return CXFA_FMToken(TOKeq); } - return std::move(m_token); + return CXFA_FMToken(TOKassign); case '<': ++m_cursor; - if (m_cursor > m_end) { - m_token->m_type = TOKlt; - return std::move(m_token); - } + if (m_cursor > m_end) + return CXFA_FMToken(TOKlt); if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } if (*m_cursor == '=') { - m_token->m_type = TOKle; ++m_cursor; - } else if (*m_cursor == '>') { - m_token->m_type = TOKne; - ++m_cursor; - } else { - m_token->m_type = TOKlt; + return CXFA_FMToken(TOKle); } - return std::move(m_token); + if (*m_cursor == '>') { + ++m_cursor; + return CXFA_FMToken(TOKne); + } + return CXFA_FMToken(TOKlt); case '>': ++m_cursor; - if (m_cursor > m_end) { - m_token->m_type = TOKgt; - return std::move(m_token); - } + if (m_cursor > m_end) + return CXFA_FMToken(TOKgt); if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } if (*m_cursor == '=') { - m_token->m_type = TOKge; ++m_cursor; - } else { - m_token->m_type = TOKgt; + return CXFA_FMToken(TOKge); } - return std::move(m_token); + return CXFA_FMToken(TOKgt); case ',': - m_token->m_type = TOKcomma; ++m_cursor; - return std::move(m_token); + return CXFA_FMToken(TOKcomma); case '(': - m_token->m_type = TOKlparen; ++m_cursor; - return std::move(m_token); + return CXFA_FMToken(TOKlparen); case ')': - m_token->m_type = TOKrparen; ++m_cursor; - return std::move(m_token); + return CXFA_FMToken(TOKrparen); case '[': - m_token->m_type = TOKlbracket; ++m_cursor; - return std::move(m_token); + return CXFA_FMToken(TOKlbracket); case ']': - m_token->m_type = TOKrbracket; ++m_cursor; - return std::move(m_token); + return CXFA_FMToken(TOKrbracket); case '&': ++m_cursor; - m_token->m_type = TOKand; - return std::move(m_token); + return CXFA_FMToken(TOKand); case '|': ++m_cursor; - m_token->m_type = TOKor; - return std::move(m_token); + return CXFA_FMToken(TOKor); case '+': ++m_cursor; - m_token->m_type = TOKplus; - return std::move(m_token); + return CXFA_FMToken(TOKplus); case '-': ++m_cursor; - m_token->m_type = TOKminus; - return std::move(m_token); + return CXFA_FMToken(TOKminus); case '*': ++m_cursor; - m_token->m_type = TOKmul; - return std::move(m_token); + return CXFA_FMToken(TOKmul); case '/': { ++m_cursor; - if (m_cursor > m_end) { - m_token->m_type = TOKdiv; - return std::move(m_token); - } + if (m_cursor > m_end) + return CXFA_FMToken(TOKdiv); if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } - if (*m_cursor != '/') { - m_token->m_type = TOKdiv; - return std::move(m_token); - } + if (*m_cursor != '/') + return CXFA_FMToken(TOKdiv); + AdvanceForComment(); break; } case '.': ++m_cursor; - if (m_cursor > m_end) { - m_token->m_type = TOKdot; - return std::move(m_token); - } + if (m_cursor > m_end) + return CXFA_FMToken(TOKdot); if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } if (*m_cursor == '.') { - m_token->m_type = TOKdotdot; ++m_cursor; - } else if (*m_cursor == '*') { - m_token->m_type = TOKdotstar; - ++m_cursor; - } else if (*m_cursor == '#') { - m_token->m_type = TOKdotscream; - ++m_cursor; - } else if (*m_cursor <= '9' && *m_cursor >= '0') { - m_token->m_type = TOKnumber; - --m_cursor; - AdvanceForNumber(); - } else { - m_token->m_type = TOKdot; + return CXFA_FMToken(TOKdotdot); } - return std::move(m_token); + if (*m_cursor == '*') { + ++m_cursor; + return CXFA_FMToken(TOKdotstar); + } + if (*m_cursor == '#') { + ++m_cursor; + return CXFA_FMToken(TOKdotscream); + } + if (*m_cursor <= '9' && *m_cursor >= '0') { + --m_cursor; + return AdvanceForNumber(); + } + return CXFA_FMToken(TOKdot); default: if (IsWhitespaceCharacter(*m_cursor)) { ++m_cursor; @@ -328,35 +295,34 @@ } if (!IsInitialIdentifierCharacter(*m_cursor)) { RaiseError(); - return nullptr; + return CXFA_FMToken(); } - AdvanceForIdentifier(); - return std::move(m_token); + return AdvanceForIdentifier(); } } - - // If there isn't currently a token type then mark it EOF. - if (m_token->m_type == TOKreserver) - m_token->m_type = TOKeof; - return std::move(m_token); + return CXFA_FMToken(TOKeof); } -void CXFA_FMLexer::AdvanceForNumber() { +CXFA_FMToken CXFA_FMLexer::AdvanceForNumber() { // This will set end to the character after the end of the number. wchar_t* end = nullptr; if (m_cursor) wcstod(const_cast<wchar_t*>(m_cursor), &end); if (!end || FXSYS_iswalpha(*end)) { RaiseError(); - return; + return CXFA_FMToken(); } - m_token->m_string = + CXFA_FMToken token(TOKnumber); + token.m_string = WideStringView(m_cursor, static_cast<size_t>(end - m_cursor)); m_cursor = end; + return token; } -void CXFA_FMLexer::AdvanceForString() { +CXFA_FMToken CXFA_FMLexer::AdvanceForString() { + CXFA_FMToken token(TOKstring); + const wchar_t* start = m_cursor; ++m_cursor; while (m_cursor <= m_end && *m_cursor) { @@ -368,9 +334,9 @@ ++m_cursor; // If the end of the input has been reached it was not escaped. if (m_cursor > m_end) { - m_token->m_string = + token.m_string = WideStringView(start, static_cast<size_t>(m_cursor - start)); - return; + return token; } // If the next character is not a " then the end of the string has been // found. @@ -378,8 +344,8 @@ if (!IsFormCalcCharacter(*m_cursor)) break; - m_token->m_string = WideStringView(start, (m_cursor - start)); - return; + token.m_string = WideStringView(start, (m_cursor - start)); + return token; } } ++m_cursor; @@ -387,24 +353,28 @@ // Didn't find the end of the string. RaiseError(); + return CXFA_FMToken(); } -void CXFA_FMLexer::AdvanceForIdentifier() { +CXFA_FMToken CXFA_FMLexer::AdvanceForIdentifier() { const wchar_t* start = m_cursor; ++m_cursor; while (m_cursor <= m_end && *m_cursor) { if (!IsFormCalcCharacter(*m_cursor)) { RaiseError(); - return; + return CXFA_FMToken(); } if (!IsIdentifierCharacter(*m_cursor)) break; ++m_cursor; } - m_token->m_string = + + WideStringView str = WideStringView(start, static_cast<size_t>(m_cursor - start)); - m_token->m_type = TokenizeIdentifier(m_token->m_string); + CXFA_FMToken token(TokenizeIdentifier(str)); + token.m_string = str; + return token; } void CXFA_FMLexer::AdvanceForComment() {
diff --git a/xfa/fxfa/fm2js/cxfa_fmlexer.h b/xfa/fxfa/fm2js/cxfa_fmlexer.h index c94f2a9..eb9e45e 100644 --- a/xfa/fxfa/fm2js/cxfa_fmlexer.h +++ b/xfa/fxfa/fm2js/cxfa_fmlexer.h
@@ -91,7 +91,8 @@ class CXFA_FMToken { public: CXFA_FMToken(); - explicit CXFA_FMToken(uint32_t line_num); + explicit CXFA_FMToken(XFA_FM_TOKEN token); + CXFA_FMToken(const CXFA_FMToken&); ~CXFA_FMToken(); #ifndef NDEBUG @@ -100,7 +101,6 @@ WideStringView m_string; XFA_FM_TOKEN m_type; - uint32_t m_line_num; }; class CXFA_FMLexer { @@ -108,27 +108,24 @@ explicit CXFA_FMLexer(const WideStringView& wsFormcalc); ~CXFA_FMLexer(); - std::unique_ptr<CXFA_FMToken> NextToken(); + CXFA_FMToken NextToken(); + uint32_t GetCurrentLine() const { return m_current_line; } void SetCurrentLine(uint32_t line) { m_current_line = line; } const wchar_t* GetPos() { return m_cursor; } void SetPos(const wchar_t* pos) { m_cursor = pos; } private: - void AdvanceForNumber(); - void AdvanceForString(); - void AdvanceForIdentifier(); + CXFA_FMToken AdvanceForNumber(); + CXFA_FMToken AdvanceForString(); + CXFA_FMToken AdvanceForIdentifier(); void AdvanceForComment(); - void RaiseError() { - m_token.reset(); - m_lexer_error = true; - } + void RaiseError() { m_lexer_error = true; } const wchar_t* m_cursor; const wchar_t* const m_end; uint32_t m_current_line; - std::unique_ptr<CXFA_FMToken> m_token; bool m_lexer_error; };
diff --git a/xfa/fxfa/fm2js/cxfa_fmlexer_unittest.cpp b/xfa/fxfa/fm2js/cxfa_fmlexer_unittest.cpp index a6ab871..00dc494 100644 --- a/xfa/fxfa/fm2js/cxfa_fmlexer_unittest.cpp +++ b/xfa/fxfa/fm2js/cxfa_fmlexer_unittest.cpp
@@ -12,86 +12,86 @@ TEST(CXFA_FMLexerTest, EmptyString) { CXFA_FMLexer lexer(L""); - std::unique_ptr<CXFA_FMToken> token = lexer.NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + CXFA_FMToken token = lexer.NextToken(); + EXPECT_EQ(TOKeof, token.m_type); } TEST(CXFA_FMLexerTest, Numbers) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"-12"); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); + CXFA_FMToken token = lexer->NextToken(); // TODO(dsinclair): Should this return -12 instead of two tokens? - EXPECT_EQ(TOKminus, token->m_type); + EXPECT_EQ(TOKminus, token.m_type); token = lexer->NextToken(); - EXPECT_EQ(L"12", token->m_string); + EXPECT_EQ(L"12", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"1.5362"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"1.5362", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"1.5362", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"0.875"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"0.875", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"0.875", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"5.56e-2"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"5.56e-2", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"5.56e-2", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"1.234E10"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"1.234E10", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"1.234E10", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"123456789.012345678"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); + EXPECT_EQ(TOKnumber, token.m_type); // TODO(dsinclair): This should round as per IEEE 64-bit values. - // EXPECT_EQ(L"123456789.01234567", token->m_string); - EXPECT_EQ(L"123456789.012345678", token->m_string); + // EXPECT_EQ(L"123456789.01234567", token.m_string); + EXPECT_EQ(L"123456789.012345678", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"99999999999999999"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); + EXPECT_EQ(TOKnumber, token.m_type); // TODO(dsinclair): This is spec'd as rounding when > 16 significant digits // prior to the exponent. - // EXPECT_EQ(L"100000000000000000", token->m_string); - EXPECT_EQ(L"99999999999999999", token->m_string); + // EXPECT_EQ(L"100000000000000000", token.m_string); + EXPECT_EQ(L"99999999999999999", token.m_string); } // The quotes are stripped in CXFA_FMStringExpression::ToJavaScript. TEST(CXFA_FMLexerTest, Strings) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"\"The cat jumped over the fence.\""); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); - EXPECT_EQ(L"\"The cat jumped over the fence.\"", token->m_string); + CXFA_FMToken token = lexer->NextToken(); + EXPECT_EQ(TOKstring, token.m_type); + EXPECT_EQ(L"\"The cat jumped over the fence.\"", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"\"\""); token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); - EXPECT_EQ(L"\"\"", token->m_string); + EXPECT_EQ(TOKstring, token.m_type); + EXPECT_EQ(L"\"\"", token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>( L"\"The message reads: \"\"Warning: Insufficient Memory\"\"\""); token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); + EXPECT_EQ(TOKstring, token.m_type); EXPECT_EQ(L"\"The message reads: \"\"Warning: Insufficient Memory\"\"\"", - token->m_string); + token.m_string); lexer = pdfium::MakeUnique<CXFA_FMLexer>( L"\"\\u0047\\u006f\\u0066\\u0069\\u0073\\u0068\\u0021\\u000d\\u000a\""); token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); + EXPECT_EQ(TOKstring, token.m_type); EXPECT_EQ( L"\"\\u0047\\u006f\\u0066\\u0069\\u0073\\u0068\\u0021\\u000d\\u000a\"", - token->m_string); + token.m_string); } // Note, 'this' is a keyword but is not matched by the lexer. @@ -161,51 +161,51 @@ for (size_t i = 0; i < FX_ArraySize(op); ++i) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(op[i].op); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); - EXPECT_EQ(op[i].token, token->m_type); + CXFA_FMToken token = lexer->NextToken(); + EXPECT_EQ(op[i].token, token.m_type); } } TEST(CXFA_FMLexerTest, Comments) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"// Empty."); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + CXFA_FMToken token = lexer->NextToken(); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"//"); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"123 // Empty.\n\"str\""); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"123", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"123", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); - EXPECT_EQ(L"\"str\"", token->m_string); + EXPECT_EQ(TOKstring, token.m_type); + EXPECT_EQ(L"\"str\"", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L";"); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"; Empty."); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"123 ;Empty.\n\"str\""); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"123", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"123", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKstring, token->m_type); - EXPECT_EQ(L"\"str\"", token->m_string); + EXPECT_EQ(TOKstring, token.m_type); + EXPECT_EQ(L"\"str\"", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); } TEST(CXFA_FMLexerTest, ValidIdentifiers) { @@ -213,43 +213,50 @@ L"a", L"an_identifier", L"_ident", L"$ident", L"!ident", L"GetAddr"}; for (const auto* ident : identifiers) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(ident); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); - EXPECT_EQ(TOKidentifier, token->m_type); - EXPECT_EQ(ident, token->m_string); + CXFA_FMToken token = lexer->NextToken(); + EXPECT_EQ(TOKidentifier, token.m_type); + EXPECT_EQ(ident, token.m_string); } } TEST(CXFA_FMLexerTest, InvalidIdentifiers) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"#a"); - EXPECT_EQ(nullptr, lexer->NextToken()); + auto token = lexer->NextToken(); + EXPECT_EQ(TOKreserver, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"1a"); - EXPECT_EQ(nullptr, lexer->NextToken()); + token = lexer->NextToken(); + EXPECT_EQ(TOKreserver, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"an@identifier"); - EXPECT_NE(nullptr, lexer->NextToken()); - EXPECT_EQ(nullptr, lexer->NextToken()); - EXPECT_EQ(nullptr, lexer->NextToken()); + token = lexer->NextToken(); + EXPECT_NE(TOKreserver, token.m_type); + token = lexer->NextToken(); + EXPECT_EQ(TOKreserver, token.m_type); + token = lexer->NextToken(); + EXPECT_EQ(TOKreserver, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"_ident@"); - EXPECT_NE(nullptr, lexer->NextToken()); - EXPECT_EQ(nullptr, lexer->NextToken()); + token = lexer->NextToken(); + EXPECT_NE(TOKreserver, token.m_type); + token = lexer->NextToken(); + EXPECT_EQ(TOKreserver, token.m_type); } TEST(CXFA_FMLexerTest, Whitespace) { auto lexer = pdfium::MakeUnique<CXFA_FMLexer>(L" \t\xc\x9\xb"); - std::unique_ptr<CXFA_FMToken> token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + CXFA_FMToken token = lexer->NextToken(); + EXPECT_EQ(TOKeof, token.m_type); lexer = pdfium::MakeUnique<CXFA_FMLexer>(L"123 \t\xc\x9\xb 456"); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"123", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"123", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKnumber, token->m_type); - EXPECT_EQ(L"456", token->m_string); + EXPECT_EQ(TOKnumber, token.m_type); + EXPECT_EQ(L"456", token.m_string); token = lexer->NextToken(); - EXPECT_EQ(TOKeof, token->m_type); + EXPECT_EQ(TOKeof, token.m_type); }
diff --git a/xfa/fxfa/fm2js/cxfa_fmparser.cpp b/xfa/fxfa/fm2js/cxfa_fmparser.cpp index e634f97..948d9eb 100644 --- a/xfa/fxfa/fm2js/cxfa_fmparser.cpp +++ b/xfa/fxfa/fm2js/cxfa_fmparser.cpp
@@ -42,8 +42,9 @@ bool CXFA_FMParser::NextToken() { if (HasError()) return false; + m_token = m_lexer->NextToken(); - while (!HasError() && m_token->m_type == TOKreserver) + while (!HasError() && m_token.m_type == TOKreserver) m_token = m_lexer->NextToken(); return !HasError(); } @@ -52,7 +53,7 @@ if (HasError()) return false; - if (m_token->m_type != op) { + if (m_token.m_type != op) { m_error = true; return false; } @@ -72,13 +73,13 @@ std::unique_ptr<CXFA_FMExpression> expr; std::vector<std::unique_ptr<CXFA_FMExpression>> expressions; while (!HasError()) { - if (m_token->m_type == TOKeof || m_token->m_type == TOKendfunc || - m_token->m_type == TOKendif || m_token->m_type == TOKelseif || - m_token->m_type == TOKelse || m_token->m_type == TOKreserver) { + if (m_token.m_type == TOKeof || m_token.m_type == TOKendfunc || + m_token.m_type == TOKendif || m_token.m_type == TOKelseif || + m_token.m_type == TOKelse || m_token.m_type == TOKreserver) { return expressions; } - expr = m_token->m_type == TOKfunc ? ParseFunction() : ParseExpression(); + expr = m_token.m_type == TOKfunc ? ParseFunction() : ParseExpression(); if (!expr) { m_error = true; break; @@ -96,37 +97,37 @@ WideStringView ident; std::vector<WideStringView> arguments; std::vector<std::unique_ptr<CXFA_FMExpression>> expressions; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } else { - ident = m_token->m_string; + ident = m_token.m_string; if (!NextToken()) return nullptr; } if (!CheckThenNext(TOKlparen)) return nullptr; - if (m_token->m_type == TOKrparen) { + if (m_token.m_type == TOKrparen) { if (!NextToken()) return nullptr; } else { while (1) { - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - arguments.push_back(m_token->m_string); + arguments.push_back(m_token.m_string); if (!NextToken()) return nullptr; - if (m_token->m_type == TOKcomma) { + if (m_token.m_type == TOKcomma) { if (!NextToken()) return nullptr; continue; } - if (m_token->m_type == TOKrparen) { + if (m_token.m_type == TOKrparen) { if (!NextToken()) return nullptr; } else { @@ -138,7 +139,7 @@ } if (!CheckThenNext(TOKdo)) return nullptr; - if (m_token->m_type == TOKendfunc) { + if (m_token.m_type == TOKendfunc) { if (!NextToken()) return nullptr; } else { @@ -157,8 +158,8 @@ return nullptr; std::unique_ptr<CXFA_FMExpression> expr; - uint32_t line = m_token->m_line_num; - switch (m_token->m_type) { + uint32_t line = m_lexer->GetCurrentLine(); + switch (m_token.m_type) { case TOKvar: expr = ParseVarExpression(); break; @@ -210,20 +211,20 @@ return nullptr; WideStringView ident; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - ident = m_token->m_string; + ident = m_token.m_string; if (!NextToken()) return nullptr; std::unique_ptr<CXFA_FMExpression> expr; - if (m_token->m_type == TOKassign) { + if (m_token.m_type == TOKassign) { if (!NextToken()) return nullptr; @@ -240,12 +241,12 @@ if (HasError()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> pExp1 = ParseLogicalOrExpression(); if (!pExp1) return nullptr; int level = 1; - while (m_token->m_type == TOKassign) { + while (m_token.m_type == TOKassign) { if (!NextToken()) return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> pExp2 = ParseLogicalOrExpression(); @@ -266,7 +267,7 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> pExp1 = ParseSimpleExpression(); if (!pExp1) return nullptr; @@ -279,13 +280,13 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseLogicalAndExpression(); if (!e1) return nullptr; for (;;) { - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKor: case TOKksor: { if (!NextToken()) @@ -314,13 +315,13 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseEqualityExpression(); if (!e1) return nullptr; for (;;) { - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKand: case TOKksand: { if (!NextToken()) @@ -348,13 +349,13 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseRelationalExpression(); if (!e1) return nullptr; for (;;) { std::unique_ptr<CXFA_FMSimpleExpression> e2; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKeq: case TOKkseq: if (!NextToken()) @@ -393,14 +394,14 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseAddtiveExpression(); if (!e1) return nullptr; for (;;) { std::unique_ptr<CXFA_FMSimpleExpression> e2; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKlt: case TOKkslt: if (!NextToken()) @@ -463,14 +464,14 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseMultiplicativeExpression(); if (!e1) return nullptr; for (;;) { std::unique_ptr<CXFA_FMSimpleExpression> e2; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKplus: if (!NextToken()) return nullptr; @@ -507,14 +508,14 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> e1 = ParseUnaryExpression(); if (!e1) return nullptr; for (;;) { std::unique_ptr<CXFA_FMSimpleExpression> e2; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKmul: if (!NextToken()) return nullptr; @@ -551,8 +552,8 @@ return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> expr; - uint32_t line = m_token->m_line_num; - switch (m_token->m_type) { + uint32_t line = m_lexer->GetCurrentLine(); + switch (m_token.m_type) { case TOKplus: if (!NextToken()) return nullptr; @@ -599,25 +600,25 @@ return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> expr; - uint32_t line = m_token->m_line_num; - switch (m_token->m_type) { + uint32_t line = m_lexer->GetCurrentLine(); + switch (m_token.m_type) { case TOKnumber: expr = - pdfium::MakeUnique<CXFA_FMNumberExpression>(line, m_token->m_string); + pdfium::MakeUnique<CXFA_FMNumberExpression>(line, m_token.m_string); if (!NextToken()) return nullptr; break; case TOKstring: expr = - pdfium::MakeUnique<CXFA_FMStringExpression>(line, m_token->m_string); + pdfium::MakeUnique<CXFA_FMStringExpression>(line, m_token.m_string); if (!NextToken()) return nullptr; break; case TOKidentifier: { - WideStringView wsIdentifier(m_token->m_string); + WideStringView wsIdentifier(m_token.m_string); if (!NextToken()) return nullptr; - if (m_token->m_type == TOKlbracket) { + if (m_token.m_type == TOKlbracket) { std::unique_ptr<CXFA_FMSimpleExpression> s = ParseIndexExpression(); if (!s) return nullptr; @@ -636,7 +637,7 @@ } case TOKif: expr = pdfium::MakeUnique<CXFA_FMIdentifierExpression>(line, - m_token->m_string); + m_token.m_string); if (!expr || !NextToken()) return nullptr; break; @@ -669,7 +670,7 @@ if (HasError()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); size_t expr_count = 0; while (1) { ++expr_count; @@ -679,28 +680,28 @@ if (expr_count > kMaxPostExpressions) return nullptr; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKlparen: { if (!NextToken()) return nullptr; std::vector<std::unique_ptr<CXFA_FMSimpleExpression>> expressions; - if (m_token->m_type != TOKrparen) { - while (m_token->m_type != TOKrparen) { + if (m_token.m_type != TOKrparen) { + while (m_token.m_type != TOKrparen) { std::unique_ptr<CXFA_FMSimpleExpression> simple_expr = ParseSimpleExpression(); if (!simple_expr) return nullptr; expressions.push_back(std::move(simple_expr)); - if (m_token->m_type == TOKcomma) { + if (m_token.m_type == TOKcomma) { if (!NextToken()) return nullptr; - } else if (m_token->m_type == TOKeof || - m_token->m_type == TOKreserver) { + } else if (m_token.m_type == TOKeof || + m_token.m_type == TOKreserver) { break; } } - if (m_token->m_type != TOKrparen) { + if (m_token.m_type != TOKrparen) { m_error = true; return nullptr; } @@ -709,7 +710,7 @@ line, std::move(expr), std::move(expressions), false); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKlbracket) + if (m_token.m_type != TOKlbracket) continue; std::unique_ptr<CXFA_FMSimpleExpression> s = ParseIndexExpression(); @@ -723,37 +724,37 @@ case TOKdot: { if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - WideStringView tempStr = m_token->m_string; - uint32_t tempLine = m_token->m_line_num; + WideStringView tempStr = m_token.m_string; + uint32_t tempLine = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type == TOKlparen) { + if (m_token.m_type == TOKlparen) { std::unique_ptr<CXFA_FMSimpleExpression> pExpCall; if (!NextToken()) return nullptr; std::vector<std::unique_ptr<CXFA_FMSimpleExpression>> expressions; - if (m_token->m_type != TOKrparen) { - while (m_token->m_type != TOKrparen) { + if (m_token.m_type != TOKrparen) { + while (m_token.m_type != TOKrparen) { std::unique_ptr<CXFA_FMSimpleExpression> exp = ParseSimpleExpression(); if (!exp) return nullptr; expressions.push_back(std::move(exp)); - if (m_token->m_type == TOKcomma) { + if (m_token.m_type == TOKcomma) { if (!NextToken()) return nullptr; - } else if (m_token->m_type == TOKeof || - m_token->m_type == TOKreserver) { + } else if (m_token.m_type == TOKeof || + m_token.m_type == TOKreserver) { break; } } - if (m_token->m_type != TOKrparen) { + if (m_token.m_type != TOKrparen) { m_error = true; return nullptr; } @@ -767,7 +768,7 @@ line, std::move(expr), std::move(pExpCall)); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKlbracket) + if (m_token.m_type != TOKlbracket) continue; std::unique_ptr<CXFA_FMSimpleExpression> s = ParseIndexExpression(); @@ -776,7 +777,7 @@ expr = pdfium::MakeUnique<CXFA_FMDotAccessorExpression>( line, std::move(expr), TOKcall, L"", std::move(s)); - } else if (m_token->m_type == TOKlbracket) { + } else if (m_token.m_type == TOKlbracket) { std::unique_ptr<CXFA_FMSimpleExpression> s = ParseIndexExpression(); if (!s) return nullptr; @@ -795,15 +796,15 @@ case TOKdotdot: { if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - WideStringView tempStr = m_token->m_string; - uint32_t tempLine = m_token->m_line_num; + WideStringView tempStr = m_token.m_string; + uint32_t tempLine = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type == TOKlbracket) { + if (m_token.m_type == TOKlbracket) { std::unique_ptr<CXFA_FMSimpleExpression> s = ParseIndexExpression(); if (!s) return nullptr; @@ -822,15 +823,15 @@ case TOKdotscream: { if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - WideStringView tempStr = m_token->m_string; - uint32_t tempLine = m_token->m_line_num; + WideStringView tempStr = m_token.m_string; + uint32_t tempLine = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKlbracket) { + if (m_token.m_type != TOKlbracket) { std::unique_ptr<CXFA_FMSimpleExpression> s = pdfium::MakeUnique<CXFA_FMIndexExpression>( tempLine, ACCESSOR_NO_INDEX, nullptr, false); @@ -868,29 +869,29 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> s; XFA_FM_AccessorIndex accessorIndex = ACCESSOR_NO_RELATIVEINDEX; std::unique_ptr<CXFA_FMSimpleExpression> pExp; - if (m_token->m_type == TOKmul) { + if (m_token.m_type == TOKmul) { pExp = pdfium::MakeUnique<CXFA_FMIndexExpression>(line, accessorIndex, std::move(s), true); if (!pExp || !NextToken()) return nullptr; - if (m_token->m_type != TOKrbracket) { + if (m_token.m_type != TOKrbracket) { m_error = true; return nullptr; } return pExp; } - if (m_token->m_type == TOKplus) { + if (m_token.m_type == TOKplus) { accessorIndex = ACCESSOR_POSITIVE_INDEX; if (!NextToken()) return nullptr; - } else if (m_token->m_type == TOKminus) { + } else if (m_token.m_type == TOKminus) { accessorIndex = ACCESSOR_NEGATIVE_INDEX; if (!NextToken()) return nullptr; @@ -898,7 +899,7 @@ s = ParseSimpleExpression(); if (!s) return nullptr; - if (m_token->m_type != TOKrbracket) { + if (m_token.m_type != TOKrbracket) { m_error = true; return nullptr; } @@ -914,18 +915,18 @@ if (!CheckThenNext(TOKlparen)) return nullptr; - if (m_token->m_type == TOKrparen) { + if (m_token.m_type == TOKrparen) { m_error = true; return nullptr; } - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::unique_ptr<CXFA_FMSimpleExpression> pExp1 = ParseLogicalOrExpression(); if (!pExp1) return nullptr; int level = 1; - while (m_token->m_type == TOKassign) { + while (m_token.m_type == TOKassign) { if (!NextToken()) return nullptr; @@ -953,11 +954,11 @@ if (HasError()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); std::vector<std::unique_ptr<CXFA_FMExpression>> expressions; while (1) { std::unique_ptr<CXFA_FMExpression> expr; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKeof: case TOKendif: case TOKelseif: @@ -994,29 +995,28 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); const wchar_t* pStartPos = m_lexer->GetPos(); if (!NextToken() || !CheckThenNext(TOKlparen)) return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> pExpression; - while (m_token->m_type != TOKrparen) { + while (m_token.m_type != TOKrparen) { pExpression = ParseSimpleExpression(); if (!pExpression) return nullptr; - if (m_token->m_type != TOKcomma) + if (m_token.m_type != TOKcomma) break; if (!NextToken()) return nullptr; } if (!CheckThenNext(TOKrparen)) return nullptr; - if (m_token->m_type != TOKthen) { + if (m_token.m_type != TOKthen) { m_lexer->SetCurrentLine(line); - auto pNewToken = pdfium::MakeUnique<CXFA_FMToken>(line); - m_token = std::move(pNewToken); - m_token->m_type = TOKidentifier; - m_token->m_string = L"if"; + + m_token = CXFA_FMToken(TOKidentifier); + m_token.m_string = L"if"; m_lexer->SetPos(pStartPos); return ParseExpExpression(); } @@ -1028,7 +1028,7 @@ return nullptr; std::unique_ptr<CXFA_FMExpression> pElseExpression; - switch (m_token->m_type) { + switch (m_token.m_type) { case TOKeof: case TOKendif: if (!CheckThenNext(TOKendif)) @@ -1065,7 +1065,7 @@ if (HasError() || !IncrementParseDepthAndCheck()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; @@ -1089,7 +1089,7 @@ if (HasError()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } @@ -1105,18 +1105,18 @@ return nullptr; WideStringView wsVariant; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - wsVariant = m_token->m_string; + wsVariant = m_token.m_string; if (!NextToken()) return nullptr; - if (m_token->m_type != TOKassign) { + if (m_token.m_type != TOKassign) { m_error = true; return nullptr; } @@ -1129,9 +1129,9 @@ return nullptr; int32_t iDirection = 0; - if (m_token->m_type == TOKupto) { + if (m_token.m_type == TOKupto) { iDirection = 1; - } else if (m_token->m_type == TOKdownto) { + } else if (m_token.m_type == TOKdownto) { iDirection = -1; } else { m_error = true; @@ -1146,7 +1146,7 @@ return nullptr; std::unique_ptr<CXFA_FMSimpleExpression> pStep; - if (m_token->m_type == TOKstep) { + if (m_token.m_type == TOKstep) { if (!NextToken()) return nullptr; pStep = ParseSimpleExpression(); @@ -1180,29 +1180,29 @@ WideStringView wsIdentifier; std::vector<std::unique_ptr<CXFA_FMSimpleExpression>> pAccessors; std::unique_ptr<CXFA_FMExpression> pList; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; - if (m_token->m_type != TOKidentifier) { + if (m_token.m_type != TOKidentifier) { m_error = true; return nullptr; } - wsIdentifier = m_token->m_string; + wsIdentifier = m_token.m_string; if (!NextToken() || !CheckThenNext(TOKin) || !CheckThenNext(TOKlparen)) return nullptr; - if (m_token->m_type == TOKrparen) { + if (m_token.m_type == TOKrparen) { m_error = true; return nullptr; } - while (m_token->m_type != TOKrparen) { + while (m_token.m_type != TOKrparen) { std::unique_ptr<CXFA_FMSimpleExpression> s = ParseSimpleExpression(); if (!s) return nullptr; pAccessors.push_back(std::move(s)); - if (m_token->m_type != TOKcomma) + if (m_token.m_type != TOKcomma) break; if (!NextToken()) return nullptr; @@ -1225,7 +1225,7 @@ if (HasError()) return nullptr; - uint32_t line = m_token->m_line_num; + uint32_t line = m_lexer->GetCurrentLine(); if (!NextToken()) return nullptr; @@ -1236,5 +1236,5 @@ } bool CXFA_FMParser::HasError() const { - return m_error || m_token == nullptr; + return m_error || m_token.m_type == TOKreserver; }
diff --git a/xfa/fxfa/fm2js/cxfa_fmparser.h b/xfa/fxfa/fm2js/cxfa_fmparser.h index c536838..3958930 100644 --- a/xfa/fxfa/fm2js/cxfa_fmparser.h +++ b/xfa/fxfa/fm2js/cxfa_fmparser.h
@@ -57,7 +57,7 @@ std::unique_ptr<CXFA_FMSimpleExpression> ParseIndexExpression(); std::unique_ptr<CXFA_FMLexer> m_lexer; - std::unique_ptr<CXFA_FMToken> m_token; + CXFA_FMToken m_token; bool m_error; unsigned long m_parse_depth; unsigned long m_max_parse_depth;