additional test

src/lexer.l

@@ -30,12 +30,28 @@ return                  { return yy::parser::token::TOK_KW_RETURN; }
 for                     { return yy::parser::token::TOK_KW_FOR; }
 in                      { return yy::parser::token::TOK_KW_IN; }
 
-\(                      { return yy::parser::token::TOK_L_PAREN; }
-\)                      { return yy::parser::token::TOK_R_PAREN; }
-\:                      { return yy::parser::token::TOK_COLON; }
+\=\=                    { return yy::parser::token::TOK_EQ; }
+\!\=                    { return yy::parser::token::TOK_NE; }
+\<                      { return yy::parser::token::TOK_LT; }
+\>                      { return yy::parser::token::TOK_GT; }
+\<\=                    { return yy::parser::token::TOK_LTE; }
+\>\=                    { return yy::parser::token::TOK_GTE; }
 \=                      { return yy::parser::token::TOK_ASSIGN; }
 \+                      { return yy::parser::token::TOK_PLUS; }
 \-                      { return yy::parser::token::TOK_MINUS; }
+\*                      { return yy::parser::token::TOK_MULTIPLY; }
+\/                      { return yy::parser::token::TOK_DIVIDE; }
+\%                      { return yy::parser::token::TOK_MODULO; }
+\(                      { return yy::parser::token::TOK_L_PAREN; }
+\)                      { return yy::parser::token::TOK_R_PAREN; }
+\[                      { return yy::parser::token::TOK_L_BRACKET; }
+\]                      { return yy::parser::token::TOK_R_BRACKET; }
+\{                      { return yy::parser::token::TOK_L_BRACE; }
+\}                      { return yy::parser::token::TOK_R_BRACE; }
+\.                      { return yy::parser::token::TOK_DOT; }
+\,                      { return yy::parser::token::TOK_COMMA; }
+\:                      { return yy::parser::token::TOK_COLON; }
+\;                      { return yy::parser::token::TOK_SEMICOLON; }
 
 \"                      { yymore(); BEGIN(STRING); }
 <STRING>\"              { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }

src/parser.y

@@ -27,7 +27,7 @@
 %token <std::string> IDENTIFIER
 %token <std::string> STRING
 %token <int> NUMBER
-%token ASSIGN PLUS MINUS L_PAREN R_PAREN COLON
+%token ASSIGN PLUS MINUS L_PAREN R_PAREN COLON EQ NE LT GT LTE GTE MULTIPLY DIVIDE MODULO L_BRACKET L_BRACE R_BRACKET R_BRACE DOT SEMICOLON COMMA
 %token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN
 %token EOF NEWLINE INDENT DEDENT
 

test/lexer/10.in

@@ -0,0 +1 @@
+a & b | c ^ d << e >> f

test/lexer/10.out

@@ -0,0 +1,13 @@
+IDENTIFIER a
+BIT_AND
+IDENTIFIER b
+BIT_OR
+IDENTIFIER c
+BIT_XOR
+IDENTIFIER d
+LSHIFT
+IDENTIFIER e
+RSHIFT
+IDENTIFIER f
+NEWLINE
+EOF

test/lexer/11.in

@@ -0,0 +1,10 @@
+x += 1
+y -= 2
+z *= 3
+a /= 4
+b %= 5
+c &= 6
+d |= 7
+e ^= 8
+f <<= 9
+g >>= 10

test/lexer/11.out

@@ -0,0 +1,41 @@
+IDENTIFIER x
+PLUS_ASSIGN
+NUMBER 1
+NEWLINE
+IDENTIFIER y
+MINUS_ASSIGN
+NUMBER 2
+NEWLINE
+IDENTIFIER z
+MULTIPLY_ASSIGN
+NUMBER 3
+NEWLINE
+IDENTIFIER a
+DIVIDE_ASSIGN
+NUMBER 4
+NEWLINE
+IDENTIFIER b
+MODULO_ASSIGN
+NUMBER 5
+NEWLINE
+IDENTIFIER c
+BIT_AND_ASSIGN
+NUMBER 6
+NEWLINE
+IDENTIFIER d
+BIT_OR_ASSIGN
+NUMBER 7
+NEWLINE
+IDENTIFIER e
+BIT_XOR_ASSIGN
+NUMBER 8
+NEWLINE
+IDENTIFIER f
+LSHIFT_ASSIGN
+NUMBER 9
+NEWLINE
+IDENTIFIER g
+RSHIFT_ASSIGN
+NUMBER 10
+NEWLINE
+EOF

test/lexer/12.in

@@ -0,0 +1 @@
+result = f(a)(b).method(c)[index]

test/lexer/12.out

@@ -0,0 +1,19 @@
+IDENTIFIER result
+ASSIGN
+IDENTIFIER f
+L_PAREN
+IDENTIFIER a
+R_PAREN
+L_PAREN
+IDENTIFIER b
+R_PAREN
+DOT
+IDENTIFIER method
+L_PAREN
+IDENTIFIER c
+R_PAREN
+L_BRACKET
+IDENTIFIER index
+R_BRACKET
+NEWLINE
+EOF

test/lexer/13.in

@@ -0,0 +1,4 @@
+a = [1, 2, 3]
+b = (x, y)
+c = { "k": v, "n": 5 }
+d = {1, 2, 3}

test/lexer/13.out

@@ -0,0 +1,41 @@
+IDENTIFIER a
+ASSIGN
+L_BRACKET
+NUMBER 1
+COMMA
+NUMBER 2
+COMMA
+NUMBER 3
+R_BRACKET
+NEWLINE
+IDENTIFIER b
+ASSIGN
+L_PAREN
+IDENTIFIER x
+COMMA
+IDENTIFIER y
+R_PAREN
+NEWLINE
+IDENTIFIER c
+ASSIGN
+L_BRACE
+STRING "k"
+COLON
+IDENTIFIER v
+COMMA
+STRING "n"
+COLON
+NUMBER 5
+R_BRACE
+NEWLINE
+IDENTIFIER d
+ASSIGN
+L_BRACE
+NUMBER 1
+COMMA
+NUMBER 2
+COMMA
+NUMBER 3
+R_BRACE
+NEWLINE
+EOF

test/lexer/14.in

@@ -0,0 +1,3 @@
+"line\nbreak"
+'quote\'test'
+"unicode ☃ snowman"

test/lexer/14.out

@@ -0,0 +1,5 @@
+STRING "line\nbreak"
+STRING 'quote\'test'
+STRING "unicode ☃ snowman"
+NEWLINE
+EOF

test/lexer/15.in

@@ -0,0 +1,4 @@
+if cond:
+    pass
+else:
+    pass

test/lexer/15.out

@@ -0,0 +1,16 @@
+KW_IF
+IDENTIFIER cond
+COLON
+NEWLINE
+INDENT
+KW_PASS
+NEWLINE
+DEDENT
+KW_ELSE
+COLON
+NEWLINE
+INDENT
+KW_PASS
+NEWLINE
+DEDENT
+EOF

test/lexer/5.out

@@ -0,0 +1,13 @@
+NUMBER 42
+NUMBER 3.14
+NUMBER 0xFF
+NUMBER 0b1010
+NEWLINE
+STRING "hello"
+STRING 'world'
+STRING """multiline"""
+NEWLINE
+STRING 'test'
+STRING "escaped\"quote"
+NEWLINE
+EOF

test/lexer/6.out

@@ -0,0 +1,52 @@
+KW_DEF
+IDENTIFIER calculate
+L_PAREN
+IDENTIFIER a
+COMMA
+IDENTIFIER b
+ASSIGN
+NUMBER 10
+R_PAREN
+COLON
+NEWLINE
+INDENT
+IDENTIFIER result
+ASSIGN
+L_PAREN
+IDENTIFIER a
+PLUS
+IDENTIFIER b
+R_PAREN
+MULTIPLY
+NUMBER 2
+NEWLINE
+KW_IF
+IDENTIFIER result
+GT
+NUMBER 100
+COLON
+NEWLINE
+INDENT
+IDENTIFIER print
+L_PAREN
+STRING "Large result:"
+COMMA
+IDENTIFIER result
+R_PAREN
+NEWLINE
+DEDENT
+KW_RETURN
+IDENTIFIER result
+NEWLINE
+DEDENT
+NEWLINE
+IDENTIFIER x
+ASSIGN
+IDENTIFIER calculate
+L_PAREN
+NUMBER 5
+COMMA
+NUMBER 15
+R_PAREN
+NEWLINE
+EOF

test/lexer/7.out

@@ -0,0 +1,21 @@
+IDENTIFIER _private
+ASSIGN
+STRING "valid"
+NEWLINE
+IDENTIFIER var123
+ASSIGN
+NUMBER 456
+NEWLINE
+IDENTIFIER CamelCase
+ASSIGN
+IDENTIFIER True
+NEWLINE
+IDENTIFIER snake_case
+ASSIGN
+IDENTIFIER False
+NEWLINE
+IDENTIFIER very_long_variable_name_123
+ASSIGN
+STRING "test"
+NEWLINE
+EOF

test/lexer/8.in

@@ -0,0 +1,4 @@
+x = -a
+y = +b
+z = ~c
+not flag

test/lexer/8.out

@@ -0,0 +1,19 @@
+IDENTIFIER x
+ASSIGN
+MINUS
+IDENTIFIER a
+NEWLINE
+IDENTIFIER y
+ASSIGN
+PLUS
+IDENTIFIER b
+NEWLINE
+IDENTIFIER z
+ASSIGN
+BIT_NOT
+IDENTIFIER c
+NEWLINE
+KW_NOT
+IDENTIFIER flag
+NEWLINE
+EOF

test/lexer/9.in

@@ -0,0 +1,2 @@
+if a and b or c:
+    pass

test/lexer/9.out

@@ -0,0 +1,13 @@
+KW_IF
+IDENTIFIER a
+KW_AND
+IDENTIFIER b
+KW_OR
+IDENTIFIER c
+COLON
+NEWLINE
+INDENT
+IDENTIFIER pass
+NEWLINE
+DEDENT
+EOF