diff --git a/lib/include/pl/core/lexer.hpp b/lib/include/pl/core/lexer.hpp index 5d482698..04775c10 100644 --- a/lib/include/pl/core/lexer.hpp +++ b/lib/include/pl/core/lexer.hpp @@ -20,8 +20,9 @@ namespace pl::core { Lexer() = default; hlp::CompileResult> lex(const api::Source *source); - size_t getLongestLineLength() const { return m_longestLineLength; } - + size_t getLongestLineLength() const { + return m_longestLineLength; // Does not include line endings + } private: [[nodiscard]] char peek(size_t p = 1) const; @@ -51,20 +52,17 @@ namespace pl::core { Token makeToken(const Token& token, size_t length = 1); static Token makeTokenAt(const Token& token, Location& location, size_t length = 1); void addToken(const Token& token); - bool hasTheLineEnded(const char &ch) { - if(ch == '\n') { - m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin); - m_line++; - m_lineBegin = m_cursor; - return true; - } - return false; - } + + bool skipLineEnding(); + + static constexpr int tabsize = 4; + std::string m_sourceCode; const api::Source* m_source = nullptr; std::vector m_tokens; size_t m_cursor = 0; u32 m_line = 0; + u32 m_tabCompensation = 0; u32 m_lineBegin = 0; size_t m_longestLineLength = 0; u32 m_errorLength = 0; diff --git a/lib/source/pl/core/lexer.cpp b/lib/source/pl/core/lexer.cpp index 60c0f4cc..de7c7328 100644 --- a/lib/source/pl/core/lexer.cpp +++ b/lib/source/pl/core/lexer.cpp @@ -51,6 +51,30 @@ namespace pl::core { return intLiteral.size(); } + inline bool Lexer::skipLineEnding() { + char ch = m_sourceCode[m_cursor]; + if (ch == '\n') { + m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation); + m_tabCompensation = 0; + m_line++; + m_lineBegin = ++m_cursor; + return true; + } + else if (ch == '\r') { + m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation); + m_tabCompensation = 0; + m_line++; + if (++m_cursor Lexer::parseCharacter() { const char& c = m_sourceCode[m_cursor++]; @@ -134,8 +158,7 @@ namespace pl::core { result += character.value(); } - if (hasTheLineEnded(m_sourceCode[m_cursor])) - m_cursor++; + skipLineEnding(); return makeTokenAt(Literal::makeString(result), location, result.size()); } @@ -156,8 +179,7 @@ namespace pl::core { result += character.value(); } - if (hasTheLineEnded(m_sourceCode[m_cursor])) - m_cursor++; + skipLineEnding(); return makeTokenAt(Literal::makeString(result), location, result.size()); } @@ -317,8 +339,7 @@ namespace pl::core { } auto len = m_cursor - begin; - if (hasTheLineEnded(m_sourceCode[m_cursor])) - m_cursor++; + skipLineEnding(); return makeTokenAt(Literal::makeComment(true, result), location, len); } @@ -335,8 +356,7 @@ namespace pl::core { } auto len = m_cursor - begin; - if (hasTheLineEnded(m_sourceCode[m_cursor])) - m_cursor++; + skipLineEnding(); return makeTokenAt(Literal::makeDocComment(false, true, result), location, len); } @@ -348,10 +368,10 @@ namespace pl::core { std::string result; m_cursor += 3; - while(true) { - hasTheLineEnded(peek(0)); + while (true) { + skipLineEnding(); - if(peek(1) == '\x00') { + if(m_cursor>=m_sourceCode.size() || peek(0)=='\x00') { m_errorLength = 1; error("Unexpected end of file while parsing multi line doc comment"); return std::nullopt; @@ -375,9 +395,9 @@ namespace pl::core { m_cursor += 2; while(true) { - hasTheLineEnded(peek(0)); + skipLineEnding(); - if(peek(1) == '\x00') { + if(m_cursor>=m_sourceCode.size() || peek(0)=='\x00') { m_errorLength = 2; error("Unexpected end of file while parsing multi line doc comment"); return std::nullopt; @@ -477,6 +497,7 @@ namespace pl::core { this->m_line = 1; this->m_lineBegin = 0; this->m_longestLineLength = 0; + this->m_tabCompensation = 0; const size_t end = this->m_sourceCode.size(); @@ -486,13 +507,19 @@ namespace pl::core { const char& c = this->m_sourceCode[this->m_cursor]; if (c == '\x00') { - m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin); + m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation); break; // end of string } if (std::isblank(c) || std::isspace(c)) { - hasTheLineEnded(c); - m_cursor++; + if (c == '\t') { + u32 column = m_tabCompensation + (m_cursor - m_lineBegin + 1); + u32 tabbedColumn = (((column - 1) / tabsize + 1) * tabsize) + 1; + m_tabCompensation += tabbedColumn - column - 1; + ++m_cursor; + } + else if (!skipLineEnding()) + ++m_cursor; continue; } @@ -593,19 +620,15 @@ namespace pl::core { peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef || directive == Token::Directive::EndIf) continue; - if (hasTheLineEnded(peek(0))) { - m_cursor++; + if (skipLineEnding()) continue; - } auto directiveValue = parseDirectiveValue(); if (directiveValue.has_value()) { addToken(directiveValue.value()); if (m_line != line || peek(0) == 0) continue; - if (hasTheLineEnded(peek(0))) { - m_cursor++; + if (skipLineEnding()) continue; - } directiveValue = parseDirectiveArgument(); if (directiveValue.has_value()) { addToken(directiveValue.value()); @@ -671,12 +694,9 @@ namespace pl::core { return false; } - Location Lexer::location() { - u32 column = m_cursor - m_lineBegin; - // There is no newline before the first line so add 1 to the column - if(m_line==1) { - column += 1; - } - return Location { m_source, m_line, column, m_errorLength }; + Location Lexer::location() + { + u32 column = m_tabCompensation + m_cursor - m_lineBegin + 1; + return Location{m_source, m_line, column, m_errorLength}; } } \ No newline at end of file