bronku/python2ruby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test 5

Browse source
This commit is contained in:
bronku 2025-11-25 16:59:43 +01:00
parent 346ea64f91
commit 2be4542c1b
2 changed files with 22 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/lexer.l
View file

@ -1,7 +1,7 @@
%option c++
%option noyywrap
%option yyclass="Scanner"
%x STRING
%x STRING_DQ STRING_SQ STRING_TDQ STRING_TSQ
%{
#include "scanner.hpp"
@ -21,7 +21,6 @@
if (action == IndentAction::Dedent) return yy::parser::token::TOK_DEDENT;
}
\n { return yy::parser::token::TOK_NEWLINE; }
def { return yy::parser::token::TOK_KW_DEF; }
@ -53,10 +52,27 @@ in { return yy::parser::token::TOK_KW_IN; }
\: { return yy::parser::token::TOK_COLON; }
\; { return yy::parser::token::TOK_SEMICOLON; }
\" { yymore(); BEGIN(STRING); }
<STRING>\" { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
<STRING>. { yymore(); }
\"\"\" { yymore(); BEGIN(STRING_TDQ); }
<STRING_TDQ>\"\"\" { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
<STRING_TDQ>. { yymore(); }
\'\'\' { yymore(); BEGIN(STRING_TSQ); }
<STRING_TSQ>\'\'\' { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
<STRING_TSQ>. { yymore(); }
\" { yymore(); BEGIN(STRING_DQ); }
<STRING_DQ>\\\" { yymore(); }
<STRING_DQ>\" { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
<STRING_DQ>. { yymore(); }
\' { yymore(); BEGIN(STRING_SQ); }
<STRING_SQ>\\\' { yymore(); }
<STRING_SQ>\' { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
<STRING_SQ>. { yymore(); }
[0-9]+\.[0-9]+ { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
0x[0-9a-fA-F]+ { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
0b[01]+ { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
[0-9]+ { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
[A-Za-z_][A-Za-z0-9_]* { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }

2
test/lexer/05.out
View file

@ -10,4 +10,4 @@ NEWLINE
STRING 'test'
STRING "escaped\"quote"
NEWLINE
EOF
EOF