additional test

test/lexer/10.in

@@ -0,0 +1 @@
+a & b | c ^ d << e >> f

test/lexer/10.out

@@ -0,0 +1,13 @@
+IDENTIFIER a
+BIT_AND
+IDENTIFIER b
+BIT_OR
+IDENTIFIER c
+BIT_XOR
+IDENTIFIER d
+LSHIFT
+IDENTIFIER e
+RSHIFT
+IDENTIFIER f
+NEWLINE
+EOF

test/lexer/11.in

@@ -0,0 +1,10 @@
+x += 1
+y -= 2
+z *= 3
+a /= 4
+b %= 5
+c &= 6
+d |= 7
+e ^= 8
+f <<= 9
+g >>= 10

test/lexer/11.out

@@ -0,0 +1,41 @@
+IDENTIFIER x
+PLUS_ASSIGN
+NUMBER 1
+NEWLINE
+IDENTIFIER y
+MINUS_ASSIGN
+NUMBER 2
+NEWLINE
+IDENTIFIER z
+MULTIPLY_ASSIGN
+NUMBER 3
+NEWLINE
+IDENTIFIER a
+DIVIDE_ASSIGN
+NUMBER 4
+NEWLINE
+IDENTIFIER b
+MODULO_ASSIGN
+NUMBER 5
+NEWLINE
+IDENTIFIER c
+BIT_AND_ASSIGN
+NUMBER 6
+NEWLINE
+IDENTIFIER d
+BIT_OR_ASSIGN
+NUMBER 7
+NEWLINE
+IDENTIFIER e
+BIT_XOR_ASSIGN
+NUMBER 8
+NEWLINE
+IDENTIFIER f
+LSHIFT_ASSIGN
+NUMBER 9
+NEWLINE
+IDENTIFIER g
+RSHIFT_ASSIGN
+NUMBER 10
+NEWLINE
+EOF

test/lexer/12.in

@@ -0,0 +1 @@
+result = f(a)(b).method(c)[index]

test/lexer/12.out

@@ -0,0 +1,19 @@
+IDENTIFIER result
+ASSIGN
+IDENTIFIER f
+L_PAREN
+IDENTIFIER a
+R_PAREN
+L_PAREN
+IDENTIFIER b
+R_PAREN
+DOT
+IDENTIFIER method
+L_PAREN
+IDENTIFIER c
+R_PAREN
+L_BRACKET
+IDENTIFIER index
+R_BRACKET
+NEWLINE
+EOF

test/lexer/13.in

@@ -0,0 +1,4 @@
+a = [1, 2, 3]
+b = (x, y)
+c = { "k": v, "n": 5 }
+d = {1, 2, 3}

test/lexer/13.out

@@ -0,0 +1,41 @@
+IDENTIFIER a
+ASSIGN
+L_BRACKET
+NUMBER 1
+COMMA
+NUMBER 2
+COMMA
+NUMBER 3
+R_BRACKET
+NEWLINE
+IDENTIFIER b
+ASSIGN
+L_PAREN
+IDENTIFIER x
+COMMA
+IDENTIFIER y
+R_PAREN
+NEWLINE
+IDENTIFIER c
+ASSIGN
+L_BRACE
+STRING "k"
+COLON
+IDENTIFIER v
+COMMA
+STRING "n"
+COLON
+NUMBER 5
+R_BRACE
+NEWLINE
+IDENTIFIER d
+ASSIGN
+L_BRACE
+NUMBER 1
+COMMA
+NUMBER 2
+COMMA
+NUMBER 3
+R_BRACE
+NEWLINE
+EOF

test/lexer/14.in

@@ -0,0 +1,3 @@
+"line\nbreak"
+'quote\'test'
+"unicode ☃ snowman"

test/lexer/14.out

@@ -0,0 +1,5 @@
+STRING "line\nbreak"
+STRING 'quote\'test'
+STRING "unicode ☃ snowman"
+NEWLINE
+EOF

test/lexer/15.in

@@ -0,0 +1,4 @@
+if cond:
+    pass
+else:
+    pass

test/lexer/15.out

@@ -0,0 +1,16 @@
+KW_IF
+IDENTIFIER cond
+COLON
+NEWLINE
+INDENT
+KW_PASS
+NEWLINE
+DEDENT
+KW_ELSE
+COLON
+NEWLINE
+INDENT
+KW_PASS
+NEWLINE
+DEDENT
+EOF

test/lexer/5.out

@@ -0,0 +1,13 @@
+NUMBER 42
+NUMBER 3.14
+NUMBER 0xFF
+NUMBER 0b1010
+NEWLINE
+STRING "hello"
+STRING 'world'
+STRING """multiline"""
+NEWLINE
+STRING 'test'
+STRING "escaped\"quote"
+NEWLINE
+EOF

test/lexer/6.out

@@ -0,0 +1,52 @@
+KW_DEF
+IDENTIFIER calculate
+L_PAREN
+IDENTIFIER a
+COMMA
+IDENTIFIER b
+ASSIGN
+NUMBER 10
+R_PAREN
+COLON
+NEWLINE
+INDENT
+IDENTIFIER result
+ASSIGN
+L_PAREN
+IDENTIFIER a
+PLUS
+IDENTIFIER b
+R_PAREN
+MULTIPLY
+NUMBER 2
+NEWLINE
+KW_IF
+IDENTIFIER result
+GT
+NUMBER 100
+COLON
+NEWLINE
+INDENT
+IDENTIFIER print
+L_PAREN
+STRING "Large result:"
+COMMA
+IDENTIFIER result
+R_PAREN
+NEWLINE
+DEDENT
+KW_RETURN
+IDENTIFIER result
+NEWLINE
+DEDENT
+NEWLINE
+IDENTIFIER x
+ASSIGN
+IDENTIFIER calculate
+L_PAREN
+NUMBER 5
+COMMA
+NUMBER 15
+R_PAREN
+NEWLINE
+EOF

test/lexer/7.out

@@ -0,0 +1,21 @@
+IDENTIFIER _private
+ASSIGN
+STRING "valid"
+NEWLINE
+IDENTIFIER var123
+ASSIGN
+NUMBER 456
+NEWLINE
+IDENTIFIER CamelCase
+ASSIGN
+IDENTIFIER True
+NEWLINE
+IDENTIFIER snake_case
+ASSIGN
+IDENTIFIER False
+NEWLINE
+IDENTIFIER very_long_variable_name_123
+ASSIGN
+STRING "test"
+NEWLINE
+EOF

test/lexer/8.in

@@ -0,0 +1,4 @@
+x = -a
+y = +b
+z = ~c
+not flag

test/lexer/8.out

@@ -0,0 +1,19 @@
+IDENTIFIER x
+ASSIGN
+MINUS
+IDENTIFIER a
+NEWLINE
+IDENTIFIER y
+ASSIGN
+PLUS
+IDENTIFIER b
+NEWLINE
+IDENTIFIER z
+ASSIGN
+BIT_NOT
+IDENTIFIER c
+NEWLINE
+KW_NOT
+IDENTIFIER flag
+NEWLINE
+EOF

test/lexer/9.in

@@ -0,0 +1,2 @@
+if a and b or c:
+    pass

test/lexer/9.out

@@ -0,0 +1,13 @@
+KW_IF
+IDENTIFIER a
+KW_AND
+IDENTIFIER b
+KW_OR
+IDENTIFIER c
+COLON
+NEWLINE
+INDENT
+IDENTIFIER pass
+NEWLINE
+DEDENT
+EOF