diff --git a/src/lexer.l b/src/lexer.l index 61b2a3e..41a34e6 100644 --- a/src/lexer.l +++ b/src/lexer.l @@ -30,12 +30,28 @@ return { return yy::parser::token::TOK_KW_RETURN; } for { return yy::parser::token::TOK_KW_FOR; } in { return yy::parser::token::TOK_KW_IN; } -\( { return yy::parser::token::TOK_L_PAREN; } -\) { return yy::parser::token::TOK_R_PAREN; } -\: { return yy::parser::token::TOK_COLON; } +\=\= { return yy::parser::token::TOK_EQ; } +\!\= { return yy::parser::token::TOK_NE; } +\< { return yy::parser::token::TOK_LT; } +\> { return yy::parser::token::TOK_GT; } +\<\= { return yy::parser::token::TOK_LTE; } +\>\= { return yy::parser::token::TOK_GTE; } \= { return yy::parser::token::TOK_ASSIGN; } \+ { return yy::parser::token::TOK_PLUS; } \- { return yy::parser::token::TOK_MINUS; } +\* { return yy::parser::token::TOK_MULTIPLY; } +\/ { return yy::parser::token::TOK_DIVIDE; } +\% { return yy::parser::token::TOK_MODULO; } +\( { return yy::parser::token::TOK_L_PAREN; } +\) { return yy::parser::token::TOK_R_PAREN; } +\[ { return yy::parser::token::TOK_L_BRACKET; } +\] { return yy::parser::token::TOK_R_BRACKET; } +\{ { return yy::parser::token::TOK_L_BRACE; } +\} { return yy::parser::token::TOK_R_BRACE; } +\. { return yy::parser::token::TOK_DOT; } +\, { return yy::parser::token::TOK_COMMA; } +\: { return yy::parser::token::TOK_COLON; } +\; { return yy::parser::token::TOK_SEMICOLON; } \" { yymore(); BEGIN(STRING); } \" { BEGIN(INITIAL); yylval->as() = yytext; return yy::parser::token::TOK_STRING; } diff --git a/src/parser.y b/src/parser.y index b525e17..347c87b 100644 --- a/src/parser.y +++ b/src/parser.y @@ -27,7 +27,7 @@ %token IDENTIFIER %token STRING %token NUMBER -%token ASSIGN PLUS MINUS L_PAREN R_PAREN COLON +%token ASSIGN PLUS MINUS L_PAREN R_PAREN COLON EQ NE LT GT LTE GTE MULTIPLY DIVIDE MODULO L_BRACKET L_BRACE R_BRACKET R_BRACE DOT SEMICOLON COMMA %token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN %token EOF NEWLINE INDENT DEDENT diff --git a/test/lexer/10.in b/test/lexer/10.in new file mode 100644 index 0000000..2829c26 --- /dev/null +++ b/test/lexer/10.in @@ -0,0 +1 @@ +a & b | c ^ d << e >> f diff --git a/test/lexer/10.out b/test/lexer/10.out new file mode 100644 index 0000000..f6c1508 --- /dev/null +++ b/test/lexer/10.out @@ -0,0 +1,13 @@ +IDENTIFIER a +BIT_AND +IDENTIFIER b +BIT_OR +IDENTIFIER c +BIT_XOR +IDENTIFIER d +LSHIFT +IDENTIFIER e +RSHIFT +IDENTIFIER f +NEWLINE +EOF diff --git a/test/lexer/11.in b/test/lexer/11.in new file mode 100644 index 0000000..b4cc89d --- /dev/null +++ b/test/lexer/11.in @@ -0,0 +1,10 @@ +x += 1 +y -= 2 +z *= 3 +a /= 4 +b %= 5 +c &= 6 +d |= 7 +e ^= 8 +f <<= 9 +g >>= 10 diff --git a/test/lexer/11.out b/test/lexer/11.out new file mode 100644 index 0000000..8ebbac7 --- /dev/null +++ b/test/lexer/11.out @@ -0,0 +1,41 @@ +IDENTIFIER x +PLUS_ASSIGN +NUMBER 1 +NEWLINE +IDENTIFIER y +MINUS_ASSIGN +NUMBER 2 +NEWLINE +IDENTIFIER z +MULTIPLY_ASSIGN +NUMBER 3 +NEWLINE +IDENTIFIER a +DIVIDE_ASSIGN +NUMBER 4 +NEWLINE +IDENTIFIER b +MODULO_ASSIGN +NUMBER 5 +NEWLINE +IDENTIFIER c +BIT_AND_ASSIGN +NUMBER 6 +NEWLINE +IDENTIFIER d +BIT_OR_ASSIGN +NUMBER 7 +NEWLINE +IDENTIFIER e +BIT_XOR_ASSIGN +NUMBER 8 +NEWLINE +IDENTIFIER f +LSHIFT_ASSIGN +NUMBER 9 +NEWLINE +IDENTIFIER g +RSHIFT_ASSIGN +NUMBER 10 +NEWLINE +EOF diff --git a/test/lexer/12.in b/test/lexer/12.in new file mode 100644 index 0000000..48b5f43 --- /dev/null +++ b/test/lexer/12.in @@ -0,0 +1 @@ +result = f(a)(b).method(c)[index] diff --git a/test/lexer/12.out b/test/lexer/12.out new file mode 100644 index 0000000..ef590b8 --- /dev/null +++ b/test/lexer/12.out @@ -0,0 +1,19 @@ +IDENTIFIER result +ASSIGN +IDENTIFIER f +L_PAREN +IDENTIFIER a +R_PAREN +L_PAREN +IDENTIFIER b +R_PAREN +DOT +IDENTIFIER method +L_PAREN +IDENTIFIER c +R_PAREN +L_BRACKET +IDENTIFIER index +R_BRACKET +NEWLINE +EOF diff --git a/test/lexer/13.in b/test/lexer/13.in new file mode 100644 index 0000000..4accc95 --- /dev/null +++ b/test/lexer/13.in @@ -0,0 +1,4 @@ +a = [1, 2, 3] +b = (x, y) +c = { "k": v, "n": 5 } +d = {1, 2, 3} diff --git a/test/lexer/13.out b/test/lexer/13.out new file mode 100644 index 0000000..445f335 --- /dev/null +++ b/test/lexer/13.out @@ -0,0 +1,41 @@ +IDENTIFIER a +ASSIGN +L_BRACKET +NUMBER 1 +COMMA +NUMBER 2 +COMMA +NUMBER 3 +R_BRACKET +NEWLINE +IDENTIFIER b +ASSIGN +L_PAREN +IDENTIFIER x +COMMA +IDENTIFIER y +R_PAREN +NEWLINE +IDENTIFIER c +ASSIGN +L_BRACE +STRING "k" +COLON +IDENTIFIER v +COMMA +STRING "n" +COLON +NUMBER 5 +R_BRACE +NEWLINE +IDENTIFIER d +ASSIGN +L_BRACE +NUMBER 1 +COMMA +NUMBER 2 +COMMA +NUMBER 3 +R_BRACE +NEWLINE +EOF diff --git a/test/lexer/14.in b/test/lexer/14.in new file mode 100644 index 0000000..81d5ea0 --- /dev/null +++ b/test/lexer/14.in @@ -0,0 +1,3 @@ +"line\nbreak" +'quote\'test' +"unicode ☃ snowman" diff --git a/test/lexer/14.out b/test/lexer/14.out new file mode 100644 index 0000000..f231594 --- /dev/null +++ b/test/lexer/14.out @@ -0,0 +1,5 @@ +STRING "line\nbreak" +STRING 'quote\'test' +STRING "unicode ☃ snowman" +NEWLINE +EOF diff --git a/test/lexer/15.in b/test/lexer/15.in new file mode 100644 index 0000000..54af015 --- /dev/null +++ b/test/lexer/15.in @@ -0,0 +1,4 @@ +if cond: + pass +else: + pass diff --git a/test/lexer/15.out b/test/lexer/15.out new file mode 100644 index 0000000..4d8a482 --- /dev/null +++ b/test/lexer/15.out @@ -0,0 +1,16 @@ +KW_IF +IDENTIFIER cond +COLON +NEWLINE +INDENT +KW_PASS +NEWLINE +DEDENT +KW_ELSE +COLON +NEWLINE +INDENT +KW_PASS +NEWLINE +DEDENT +EOF diff --git a/test/lexer/5.out b/test/lexer/5.out new file mode 100644 index 0000000..2bd6e3a --- /dev/null +++ b/test/lexer/5.out @@ -0,0 +1,13 @@ +NUMBER 42 +NUMBER 3.14 +NUMBER 0xFF +NUMBER 0b1010 +NEWLINE +STRING "hello" +STRING 'world' +STRING """multiline""" +NEWLINE +STRING 'test' +STRING "escaped\"quote" +NEWLINE +EOF diff --git a/test/lexer/6.out b/test/lexer/6.out new file mode 100644 index 0000000..a482919 --- /dev/null +++ b/test/lexer/6.out @@ -0,0 +1,52 @@ +KW_DEF +IDENTIFIER calculate +L_PAREN +IDENTIFIER a +COMMA +IDENTIFIER b +ASSIGN +NUMBER 10 +R_PAREN +COLON +NEWLINE +INDENT +IDENTIFIER result +ASSIGN +L_PAREN +IDENTIFIER a +PLUS +IDENTIFIER b +R_PAREN +MULTIPLY +NUMBER 2 +NEWLINE +KW_IF +IDENTIFIER result +GT +NUMBER 100 +COLON +NEWLINE +INDENT +IDENTIFIER print +L_PAREN +STRING "Large result:" +COMMA +IDENTIFIER result +R_PAREN +NEWLINE +DEDENT +KW_RETURN +IDENTIFIER result +NEWLINE +DEDENT +NEWLINE +IDENTIFIER x +ASSIGN +IDENTIFIER calculate +L_PAREN +NUMBER 5 +COMMA +NUMBER 15 +R_PAREN +NEWLINE +EOF diff --git a/test/lexer/7.out b/test/lexer/7.out new file mode 100644 index 0000000..bb61e99 --- /dev/null +++ b/test/lexer/7.out @@ -0,0 +1,21 @@ +IDENTIFIER _private +ASSIGN +STRING "valid" +NEWLINE +IDENTIFIER var123 +ASSIGN +NUMBER 456 +NEWLINE +IDENTIFIER CamelCase +ASSIGN +IDENTIFIER True +NEWLINE +IDENTIFIER snake_case +ASSIGN +IDENTIFIER False +NEWLINE +IDENTIFIER very_long_variable_name_123 +ASSIGN +STRING "test" +NEWLINE +EOF diff --git a/test/lexer/8.in b/test/lexer/8.in new file mode 100644 index 0000000..5dcd41f --- /dev/null +++ b/test/lexer/8.in @@ -0,0 +1,4 @@ +x = -a +y = +b +z = ~c +not flag diff --git a/test/lexer/8.out b/test/lexer/8.out new file mode 100644 index 0000000..098abc4 --- /dev/null +++ b/test/lexer/8.out @@ -0,0 +1,19 @@ +IDENTIFIER x +ASSIGN +MINUS +IDENTIFIER a +NEWLINE +IDENTIFIER y +ASSIGN +PLUS +IDENTIFIER b +NEWLINE +IDENTIFIER z +ASSIGN +BIT_NOT +IDENTIFIER c +NEWLINE +KW_NOT +IDENTIFIER flag +NEWLINE +EOF diff --git a/test/lexer/9.in b/test/lexer/9.in new file mode 100644 index 0000000..039adb7 --- /dev/null +++ b/test/lexer/9.in @@ -0,0 +1,2 @@ +if a and b or c: + pass diff --git a/test/lexer/9.out b/test/lexer/9.out new file mode 100644 index 0000000..cde813d --- /dev/null +++ b/test/lexer/9.out @@ -0,0 +1,13 @@ +KW_IF +IDENTIFIER a +KW_AND +IDENTIFIER b +KW_OR +IDENTIFIER c +COLON +NEWLINE +INDENT +IDENTIFIER pass +NEWLINE +DEDENT +EOF