Skip to content

Accurate column error info with tabs. Avoid copy of pattern string. #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open

Accurate column error info with tabs. Avoid copy of pattern string. #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
51a6d74
Add code from dev branch
shewitt-au Jul 10, 2025
f9e9304
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
95e0ccf
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
f52b404
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
4ecaf94
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
09bdf7a
Fix auto code-formatter disaster
shewitt-au Jul 11, 2025
35a70d9
Fix build error.
shewitt-au Jul 11, 2025
8c37d1d
Make whole longest line visible
shewitt-au Jul 11, 2025
dd10d49
Fix grammar in comment
shewitt-au Jul 11, 2025
c021818
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
155dcbe
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
0c71626
Merge branch 'accurate-column' of https://github.com/shewitt-au/Steve…
shewitt-au Jul 12, 2025
b1872f6
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
b4fab84
Remove unwanted formatting changes
shewitt-au Jul 12, 2025
6dc69fa
Fixed potential buffer overrun
shewitt-au Jul 12, 2025
fefa424
Fixed bug while investigating feedback from @paxcut
shewitt-au Jul 12, 2025
c4373e7
Longest line length
shewitt-au Jul 13, 2025
f77e420
Merge branch 'master' into accurate-column
shewitt-au Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 9 additions & 11 deletions lib/include/pl/core/lexer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ namespace pl::core {
Lexer() = default;

hlp::CompileResult<std::vector<Token>> lex(const api::Source *source);
size_t getLongestLineLength() const { return m_longestLineLength; }

size_t getLongestLineLength() const {
return m_longestLineLength; // Does not include line endings
}

private:
[[nodiscard]] char peek(size_t p = 1) const;
Expand Down Expand Up @@ -51,20 +52,17 @@ namespace pl::core {
Token makeToken(const Token& token, size_t length = 1);
static Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
void addToken(const Token& token);
bool hasTheLineEnded(const char &ch) {
if(ch == '\n') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);
m_line++;
m_lineBegin = m_cursor;
return true;
}
return false;
}

bool skipLineEnding();

static constexpr int tabsize = 4;

std::string m_sourceCode;
const api::Source* m_source = nullptr;
std::vector<Token> m_tokens;
size_t m_cursor = 0;
u32 m_line = 0;
u32 m_tabCompensation = 0;
u32 m_lineBegin = 0;
size_t m_longestLineLength = 0;
u32 m_errorLength = 0;
Expand Down
78 changes: 49 additions & 29 deletions lib/source/pl/core/lexer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,30 @@ namespace pl::core {
return intLiteral.size();
}

inline bool Lexer::skipLineEnding() {
char ch = m_sourceCode[m_cursor];
if (ch == '\n') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
m_tabCompensation = 0;
m_line++;
m_lineBegin = ++m_cursor;
return true;
}
else if (ch == '\r') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
m_tabCompensation = 0;
m_line++;
if (++m_cursor<m_sourceCode.size()) {
ch = m_sourceCode[m_cursor];
if (ch == '\n')
++m_cursor;
}
m_lineBegin = m_cursor;
return true;
}

return false;
}

std::optional<char> Lexer::parseCharacter() {
const char& c = m_sourceCode[m_cursor++];
Expand Down Expand Up @@ -134,8 +158,7 @@ namespace pl::core {
result += character.value();
}

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeString(result), location, result.size());
}
Expand All @@ -156,8 +179,7 @@ namespace pl::core {
result += character.value();
}

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeString(result), location, result.size());
}
Expand Down Expand Up @@ -317,8 +339,7 @@ namespace pl::core {
}
auto len = m_cursor - begin;

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeComment(true, result), location, len);
}
Expand All @@ -335,8 +356,7 @@ namespace pl::core {
}
auto len = m_cursor - begin;

if (hasTheLineEnded(m_sourceCode[m_cursor]))
m_cursor++;
skipLineEnding();

return makeTokenAt(Literal::makeDocComment(false, true, result), location, len);
}
Expand All @@ -348,10 +368,10 @@ namespace pl::core {
std::string result;

m_cursor += 3;
while(true) {
hasTheLineEnded(peek(0));
while (true) {
skipLineEnding();

if(peek(1) == '\x00') {
if(m_cursor>=m_sourceCode.size() || peek(0)=='\x00') {
m_errorLength = 1;
error("Unexpected end of file while parsing multi line doc comment");
return std::nullopt;
Expand All @@ -375,9 +395,9 @@ namespace pl::core {

m_cursor += 2;
while(true) {
hasTheLineEnded(peek(0));
skipLineEnding();

if(peek(1) == '\x00') {
if(m_cursor>=m_sourceCode.size() || peek(0)=='\x00') {
m_errorLength = 2;
error("Unexpected end of file while parsing multi line doc comment");
return std::nullopt;
Expand Down Expand Up @@ -477,6 +497,7 @@ namespace pl::core {
this->m_line = 1;
this->m_lineBegin = 0;
this->m_longestLineLength = 0;
this->m_tabCompensation = 0;

const size_t end = this->m_sourceCode.size();

Expand All @@ -486,13 +507,19 @@ namespace pl::core {
const char& c = this->m_sourceCode[this->m_cursor];

if (c == '\x00') {
m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);
m_longestLineLength = std::max(m_longestLineLength, m_cursor-m_lineBegin+m_tabCompensation);
break; // end of string
}

if (std::isblank(c) || std::isspace(c)) {
hasTheLineEnded(c);
m_cursor++;
if (c == '\t') {
u32 column = m_tabCompensation + (m_cursor - m_lineBegin + 1);
u32 tabbedColumn = (((column - 1) / tabsize + 1) * tabsize) + 1;
m_tabCompensation += tabbedColumn - column - 1;
++m_cursor;
}
else if (!skipLineEnding())
++m_cursor;
continue;
}

Expand Down Expand Up @@ -593,19 +620,15 @@ namespace pl::core {
peek(0) == 0 || directive == Token::Directive::IfDef || directive == Token::Directive::IfNDef ||
directive == Token::Directive::EndIf)
continue;
if (hasTheLineEnded(peek(0))) {
m_cursor++;
if (skipLineEnding())
continue;
}
auto directiveValue = parseDirectiveValue();
if (directiveValue.has_value()) {
addToken(directiveValue.value());
if (m_line != line || peek(0) == 0)
continue;
if (hasTheLineEnded(peek(0))) {
m_cursor++;
if (skipLineEnding())
continue;
}
directiveValue = parseDirectiveArgument();
if (directiveValue.has_value()) {
addToken(directiveValue.value());
Expand Down Expand Up @@ -671,12 +694,9 @@ namespace pl::core {
return false;
}

Location Lexer::location() {
u32 column = m_cursor - m_lineBegin;
// There is no newline before the first line so add 1 to the column
if(m_line==1) {
column += 1;
}
return Location { m_source, m_line, column, m_errorLength };
Location Lexer::location()
{
u32 column = m_tabCompensation + m_cursor - m_lineBegin + 1;
return Location{m_source, m_line, column, m_errorLength};
}
}
Loading