Skip to content

Commit 2921b88

Browse files
Program132Program132
committed
Token Management
1 parent ee5faf7 commit 2921b88

File tree

2 files changed

+251
-0
lines changed

2 files changed

+251
-0
lines changed

‎src/TokenBuilding.cpp‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
#include "TokenBuilding.h"
2+
3+
namespace FPL {
4+
std::vector<Token> TokenBuilding::parseToken(const std::string &content) {
5+
std::vector<Token> tokens;
6+
Token currentToken;
7+
8+
for (auto &currT : content) // char
9+
{
10+
if (currentToken.mType == CHAINE_ESPACE_SEQUENCE) {
11+
switch(currT) {
12+
case 'n':
13+
currentToken.mText.append(1, '\n');
14+
break;
15+
case 'r':
16+
currentToken.mText.append(1, '\r');
17+
break;
18+
case 't':
19+
currentToken.mText.append(1, '\t');
20+
break;
21+
case '\\':
22+
currentToken.mText.append(1, '\\');
23+
break;
24+
default:
25+
throw std::runtime_error(std::string("Inconnu : \\") + std::string(1, currT) +
26+
" dans la ligne " + std::to_string(currentToken.mLineNumber) + ".");
27+
}
28+
29+
currentToken.mType = CHAINE_LITERAL;
30+
continue;
31+
}
32+
else if (currentToken.mType == PEUTETRE_COMMENTAIRE && currT != '/') {
33+
currentToken.mType = OPERATEUR;
34+
endToken(currentToken, tokens);
35+
continue;
36+
}
37+
38+
switch(currT)
39+
{
40+
case '0':
41+
case '1':
42+
case '2':
43+
case '3':
44+
case '4':
45+
case '5':
46+
case '6':
47+
case '7':
48+
case '8':
49+
case '9':
50+
if (currentToken.mType == ESPACEVIDE) {
51+
currentToken.mType = ENTIER_LITERAL;
52+
currentToken.mText.append(1, currT);
53+
} else if (currentToken.mType == PEUTETRE_DOUBLE) {
54+
currentToken.mType = DECIMAL_LITERAL;
55+
currentToken.mText.append(1, currT);
56+
} else {
57+
currentToken.mText.append(1, currT);
58+
}
59+
break;
60+
61+
62+
case '.':
63+
if (currentToken.mType == ESPACEVIDE) {
64+
currentToken.mType = PEUTETRE_DOUBLE;
65+
currentToken.mText.append(1, currT);
66+
} else if (currentToken.mType == ENTIER_LITERAL) {
67+
currentToken.mType = DECIMAL_LITERAL;
68+
currentToken.mText.append(1, currT);
69+
} else if (currentToken.mType == CHAINE_LITERAL) {
70+
currentToken.mText.append(1, currT);
71+
} else {
72+
endToken( currentToken, tokens);
73+
currentToken.mType = OPERATEUR;
74+
currentToken.mText.append(1, currT);
75+
endToken(currentToken, tokens);
76+
}
77+
break;
78+
79+
80+
case '{':
81+
case '}':
82+
case '(':
83+
case ')':
84+
case '=':
85+
case '+':
86+
case '-':
87+
case '*':
88+
case '>':
89+
case '<':
90+
case ';':
91+
case ',':
92+
if (currentToken.mType != CHAINE_LITERAL) {
93+
endToken(currentToken, tokens);
94+
currentToken.mType = OPERATEUR;
95+
currentToken.mText.append(1, currT);
96+
endToken(currentToken, tokens);
97+
} else {
98+
currentToken.mText.append(1, currT);
99+
}
100+
break;
101+
102+
103+
case '\t':
104+
case ' ':
105+
if (currentToken.mType == CHAINE_LITERAL || currentToken.mType == COMMENTAIRE)
106+
{
107+
currentToken.mText.append(1, currT);
108+
}
109+
else {
110+
endToken(currentToken, tokens);
111+
}
112+
break;
113+
114+
115+
case '\r':
116+
case '\n':
117+
endToken(currentToken, tokens);
118+
++currentToken.mLineNumber;
119+
break;
120+
121+
122+
case '"':
123+
if (currentToken.mType != CHAINE_LITERAL) {
124+
endToken(currentToken, tokens);
125+
currentToken.mType = CHAINE_LITERAL;
126+
currentToken.mText.append(1, currT);
127+
} else if (currentToken.mType == CHAINE_LITERAL) {
128+
endToken(currentToken, tokens);
129+
}
130+
break;
131+
132+
133+
case '\\':
134+
if (currentToken.mType == CHAINE_LITERAL) {
135+
endToken(currentToken, tokens);
136+
currentToken.mType = CHAINE_LITERAL;
137+
currentToken.mText.append(1, currT);
138+
} else {
139+
endToken(currentToken, tokens);
140+
currentToken.mType = OPERATEUR;
141+
currentToken.mText.append(1, currT);
142+
endToken(currentToken, tokens);
143+
}
144+
break;
145+
146+
case '/':
147+
if (currentToken.mType == CHAINE_LITERAL) {
148+
currentToken.mText.append(1, currT);
149+
} else if (currentToken.mType == PEUTETRE_COMMENTAIRE) {
150+
currentToken.mType = COMMENTAIRE;
151+
currentToken.mText.erase();
152+
} else {
153+
endToken(currentToken, tokens);
154+
currentToken.mType = PEUTETRE_COMMENTAIRE;
155+
currentToken.mText.append(1, currT);
156+
}
157+
break;
158+
159+
default:
160+
if (currentToken.mType == ESPACEVIDE || currentToken.mType == ENTIER_LITERAL
161+
|| currentToken.mType == DECIMAL_LITERAL)
162+
{
163+
endToken(currentToken, tokens);
164+
currentToken.mType = IDENTIFIANT;
165+
currentToken.mText.append(1, currT);
166+
}
167+
else {
168+
currentToken.mText.append(1, currT);
169+
}
170+
break;
171+
}
172+
}
173+
174+
return tokens;
175+
}
176+
177+
void TokenBuilding::endToken(Token &token, std::vector<Token> &tokens) {
178+
if (token.mType == COMMENTAIRE) {
179+
//std::cout << "Commentaire ignoree : '" << token.mText << "'." << std::endl;
180+
} else if (token.mType != ESPACEVIDE) {
181+
tokens.push_back(token);
182+
}
183+
if (token.mType == PEUTETRE_DOUBLE)
184+
{
185+
if (token.mText == ".") {
186+
token.mType = OPERATEUR;
187+
}
188+
else {
189+
token.mType = DECIMAL_LITERAL;
190+
}
191+
}
192+
token.mType = ESPACEVIDE;
193+
token.mText.erase();
194+
}
195+
196+
[[maybe_unused]] void Token::DebugPrint() const {
197+
std::cout << "Token(" << sTokenTypes[mType] << ", '" << mText << "'." << std::endl;
198+
}
199+
}

‎src/TokenBuilding.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#pragma once
2+
3+
#include <iostream>
4+
#include <vector>
5+
#include <string>
6+
#include <stdexcept>
7+
8+
namespace FPL {
9+
enum TokenTypes {
10+
ESPACEVIDE,
11+
IDENTIFIANT,
12+
COMMENTAIRE,
13+
PEUTETRE_COMMENTAIRE,
14+
ENTIER_LITERAL, // Ints
15+
CHAINE_LITERAL, // Strings
16+
OPERATEUR, // Opérateur comme '+', '-', '*', '/', '(', '{', '[', ',',
17+
CHAINE_ESPACE_SEQUENCE,
18+
PEUTETRE_DOUBLE,
19+
DECIMAL_LITERAL
20+
};
21+
22+
static const char *sTokenTypes[] = {
23+
"ESPACEVIDE",
24+
"IDENTIFIANT",
25+
"COMMENTAIRE",
26+
"PEUTETRE_COMMENTAIRE",
27+
"ENTIER_LITERAL",
28+
"CHAINE_LITERAL",
29+
"OPERATEUR",
30+
"CHAINE_ESPACE_SEQUENCE",
31+
"PEUTETRE_DOUBLE",
32+
"DECIMAL_LITERAL"
33+
};
34+
35+
class Token {
36+
public:
37+
enum TokenTypes mType; // Type du token
38+
std::string mText; // Contenue du token
39+
size_t mLineNumber{1}; // Ligne de code
40+
41+
[[maybe_unused]] void DebugPrint() const;
42+
};
43+
44+
class TokenBuilding {
45+
public:
46+
std::vector<Token> parseToken(const std::string &content);
47+
48+
49+
private:
50+
static void endToken(Token &token, std::vector<Token> &tokens);
51+
};
52+
}

0 commit comments

Comments
 (0)