parser start

makefile

@@ -11,7 +11,7 @@ $(TARGET): build/parser.o build/lexer.o build/main.o
 build/parser.cpp build/parser.hpp: src/parser.y | build
 	$(BISON) -d -o build/parser.cpp src/parser.y
 
-build/lexer.cpp build/lexer.hpp: src/lexer.l build/parser.hpp src/scanner.hpp src/indent_stack.hpp| build
+build/lexer.cpp build/lexer.hpp: src/lexer.l build/parser.hpp src/scanner.hpp src/indent_stack.hpp build
 	$(FLEX) --header-file=build/lexer.hpp -o build/lexer.cpp src/lexer.l
 
 build/%.o: build/%.cpp
@@ -20,6 +20,8 @@ build/%.o: build/%.cpp
 build/%.o: src/%.cpp | build
 	$(CXX) -c $(ARGS) $< -o $@
 
+src/ast.hpp: src/ast/base.hpp src/ast/expressions.hpp src/ast/literals.hpp src/ast/operators.hpp src/ast/statements.hpp src/ast/string_utils.hpp
+
 build:
 	mkdir -p build
 

src/ast.hpp

@@ -0,0 +1,7 @@
+#pragma once
+#include "ast/string_utils.hpp"
+#include "ast/base.hpp"
+#include "ast/expressions.hpp"
+#include "ast/literals.hpp"
+#include "ast/operators.hpp"
+#include "ast/statements.hpp"

src/ast/base.hpp

@@ -0,0 +1,15 @@
+#pragma once
+#include <string>
+
+struct Node
+{
+    virtual ~Node() = default;
+    virtual std::string dump(int indent = 0) const = 0;
+};
+
+struct Expr : Node
+{
+};
+struct Stmt : Node
+{
+};

src/ast/expressions.hpp

@@ -0,0 +1,145 @@
+#pragma once
+#include <sstream>
+#include "base.hpp"
+#include "operators.hpp"
+#include "literals.hpp"
+
+template <typename T>
+using ptr = std::unique_ptr<T>;
+
+struct BinOp : Expr
+{
+    BinaryOperator op;
+    ptr<Expr> left;
+    ptr<Expr> right;
+
+    BinOp(BinaryOperator o, ptr<Expr> l, ptr<Expr> r)
+        : op(o), left(std::move(l)), right(std::move(r)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "BinOp(" << op.symbol() << ")\n";
+        oss << left->dump(indent + 1) << "\n";
+        oss << right->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct UnaryOpExpr : Expr
+{
+    UnaryOperator op;
+    ptr<Expr> operand;
+
+    UnaryOpExpr(UnaryOperator o, ptr<Expr> expr)
+        : op(o), operand(std::move(expr)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "UnaryOp(" << op.symbol() << ")\n";
+        oss << operand->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct Call : Expr
+{
+    ptr<Expr> func;
+    std::vector<ptr<Expr>> args;
+
+    Call(ptr<Expr> f, std::vector<ptr<Expr>> a)
+        : func(std::move(f)), args(std::move(a)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Call\n";
+        oss << func->dump(indent + 1);
+        for (const auto &arg : args)
+        {
+            oss << "\n"
+                << arg->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct Attribute : Expr
+{
+    ptr<Expr> value;
+    std::string attr;
+
+    Attribute(ptr<Expr> v, std::string a)
+        : value(std::move(v)), attr(a) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Attribute(." << attr << ")\n";
+        oss << value->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct Subscript : Expr
+{
+    ptr<Expr> value;
+    ptr<Expr> index;
+
+    Subscript(ptr<Expr> v, ptr<Expr> i)
+        : value(std::move(v)), index(std::move(i)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Subscript\n";
+        oss << value->dump(indent + 1) << "\n";
+        oss << index->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct List : Expr
+{
+    std::vector<ptr<Expr>> elements;
+
+    List(std::vector<ptr<Expr>> elems)
+        : elements(std::move(elems)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "List";
+        for (const auto &elem : elements)
+        {
+            oss << "\n"
+                << elem->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct Dict : Expr
+{
+    std::vector<std::pair<ptr<Expr>, ptr<Expr>>> pairs;
+
+    Dict(std::vector<std::pair<ptr<Expr>, ptr<Expr>>> p)
+        : pairs(std::move(p)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Dict";
+        for (const auto &[key, val] : pairs)
+        {
+            oss << "\n"
+                << indent_str(indent + 1) << "Pair";
+            oss << "\n"
+                << key->dump(indent + 2);
+            oss << "\n"
+                << val->dump(indent + 2);
+        }
+        return oss.str();
+    }
+};

src/ast/literals.hpp

@@ -0,0 +1,39 @@
+#pragma once
+#include "base.hpp"
+#include "string_utils.hpp"
+
+struct Identifier : Expr
+{
+    std::string name;
+
+    Identifier(std::string n) : name(n) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        return indent_str(indent) + "Identifier(" + name + ")";
+    }
+};
+
+struct Number : Expr
+{
+    std::string value;
+
+    Number(std::string v) : value(v) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        return indent_str(indent) + "Number(" + value + ")";
+    }
+};
+
+struct String : Expr
+{
+    std::string value;
+
+    String(std::string v) : value(v) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        return indent_str(indent) + "String(\"" + value + "\")";
+    }
+};

src/ast/operators.hpp

@@ -0,0 +1,105 @@
+#pragma once
+#include <string>
+
+struct BinaryOperator
+{
+    enum Kind
+    {
+        ADD,
+        SUB,
+        MUL,
+        DIV,
+        MOD,
+        EQ,
+        NE,
+        LT,
+        LTE,
+        GT,
+        GTE,
+        AND,
+        OR,
+        BIT_AND,
+        BIT_OR,
+        BIT_XOR,
+        LSHIFT,
+        RSHIFT
+    };
+
+    Kind kind;
+
+    BinaryOperator(Kind k) : kind(k) {}
+
+    std::string symbol() const
+    {
+        switch (kind)
+        {
+        case ADD:
+            return "+";
+        case SUB:
+            return "-";
+        case MUL:
+            return "*";
+        case DIV:
+            return "/";
+        case MOD:
+            return "%";
+        case EQ:
+            return "==";
+        case NE:
+            return "!=";
+        case LT:
+            return "<";
+        case LTE:
+            return "<=";
+        case GT:
+            return ">";
+        case GTE:
+            return ">=";
+        case AND:
+            return "and";
+        case OR:
+            return "or";
+        case BIT_AND:
+            return "&";
+        case BIT_OR:
+            return "|";
+        case BIT_XOR:
+            return "^";
+        case LSHIFT:
+            return "<<";
+        case RSHIFT:
+            return ">>";
+        default:
+            return "?";
+        }
+    }
+};
+
+struct UnaryOperator
+{
+    enum Kind
+    {
+        NOT,
+        NEG,
+        BIT_NOT
+    };
+
+    Kind kind;
+
+    UnaryOperator(Kind k) : kind(k) {}
+
+    std::string symbol() const
+    {
+        switch (kind)
+        {
+        case NOT:
+            return "not";
+        case NEG:
+            return "-";
+        case BIT_NOT:
+            return "~";
+        default:
+            return "?";
+        }
+    }
+};

src/ast/statements.hpp

@@ -0,0 +1,192 @@
+#pragma once
+#include <iostream>
+#include <sstream>
+#include "string_utils.hpp"
+#include "base.hpp"
+
+template <typename T>
+using ptr = std::unique_ptr<T>;
+
+struct Assign : Stmt
+{
+    ptr<Expr> target;
+    ptr<Expr> value;
+
+    Assign(ptr<Expr> t, ptr<Expr> v)
+        : target(std::move(t)), value(std::move(v)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Assign\n";
+        oss << target->dump(indent + 1) << "\n";
+        oss << value->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct ExprStmt : Stmt
+{
+    ptr<Expr> expr;
+
+    ExprStmt(ptr<Expr> e) : expr(std::move(e)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "ExprStmt\n";
+        oss << expr->dump(indent + 1);
+        return oss.str();
+    }
+};
+
+struct Return : Stmt
+{
+    ptr<Expr> value;
+
+    Return(ptr<Expr> v) : value(std::move(v)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "Return\n";
+        if (value)
+        {
+            oss << value->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct Pass : Stmt
+{
+    std::string dump(int indent = 0) const override
+    {
+        return indent_str(indent) + "Pass";
+    }
+};
+
+struct If : Stmt
+{
+    ptr<Expr> condition;
+    std::vector<ptr<Stmt>> then_body;
+    std::vector<ptr<Stmt>> else_body;
+
+    If(ptr<Expr> cond, std::vector<ptr<Stmt>> then_b, std::vector<ptr<Stmt>> else_b)
+        : condition(std::move(cond)),
+          then_body(std::move(then_b)),
+          else_body(std::move(else_b)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "If\n";
+        oss << condition->dump(indent + 1);
+        for (const auto &stmt : then_body)
+        {
+            oss << "\n"
+                << stmt->dump(indent + 1);
+        }
+        if (!else_body.empty())
+        {
+            oss << "\n"
+                << indent_str(indent + 1) << "Else";
+            for (const auto &stmt : else_body)
+            {
+                oss << "\n"
+                    << stmt->dump(indent + 1);
+            }
+        }
+        return oss.str();
+    }
+};
+
+struct For : Stmt
+{
+    std::string target;
+    ptr<Expr> iter;
+    std::vector<ptr<Stmt>> body;
+
+    For(std::string t, ptr<Expr> i, std::vector<ptr<Stmt>> b)
+        : target(t), iter(std::move(i)), body(std::move(b)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "For(" << target << ")\n";
+        oss << iter->dump(indent + 1);
+        for (const auto &stmt : body)
+        {
+            oss << "\n"
+                << stmt->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct FunctionDef : Stmt
+{
+    std::string name;
+    std::vector<std::string> params;
+    std::vector<ptr<Stmt>> body;
+
+    FunctionDef(std::string n, std::vector<std::string> p, std::vector<ptr<Stmt>> b)
+        : name(n), params(std::move(p)), body(std::move(b)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "FunctionDef(" << name << ")";
+        for (const auto &param : params)
+        {
+            oss << "\n"
+                << indent_str(indent + 1) << "Param(" << param << ")";
+        }
+        for (const auto &stmt : body)
+        {
+            oss << "\n"
+                << stmt->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct ClassDef : Stmt
+{
+    std::string name;
+    std::vector<ptr<Stmt>> body;
+
+    ClassDef(std::string n, std::vector<ptr<Stmt>> b)
+        : name(n), body(std::move(b)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << indent_str(indent) << "ClassDef(" << name << ")";
+        for (const auto &stmt : body)
+        {
+            oss << "\n"
+                << stmt->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};
+
+struct Module : Node
+{
+    std::vector<ptr<Stmt>> body;
+
+    Module(std::vector<ptr<Stmt>> b) : body(std::move(b)) {}
+
+    std::string dump(int indent = 0) const override
+    {
+        std::ostringstream oss;
+        oss << "Module";
+        for (const auto &stmt : body)
+        {
+            oss << "\n"
+                << stmt->dump(indent + 1);
+        }
+        return oss.str();
+    }
+};

src/ast/string_utils.hpp

@@ -0,0 +1,7 @@
+#pragma once
+#include <string>
+
+inline std::string indent_str(int level)
+{
+    return std::string(level * 2, ' ');
+}

src/main.cpp

@@ -1,5 +1,6 @@
 #include "parser.hpp"
 #include "scanner.hpp"
+#include "ast.hpp"
 #include <iostream>
 #include <fstream>
 
@@ -36,6 +37,7 @@ int main(int argc, char *argv[])
     std::ifstream file_stream;
 
     bool token_dump = false;
+    bool ast_dump = false;
 
     for (int i = 1; i < argc; ++i)
     {
@@ -49,6 +51,10 @@ int main(int argc, char *argv[])
         {
             token_dump = true;
         }
+        else if (arg == "--dump-ast")
+        {
+            ast_dump = true;
+        }
     }
 
     Scanner scanner(*input_stream);
@@ -57,6 +63,13 @@ int main(int argc, char *argv[])
         dump_tokens(scanner);
         return 0;
     }
-    yy::parser parser(&scanner);
+
-    return parser.parse();
+    std::unique_ptr<Module> ast_root;
+    yy::parser parser(&scanner, &ast_root);
+    parser.parse();
+
+    if (ast_dump)
+    {
+        std::cout << ast_root->dump();
+    }
 }

src/parser.y

@@ -11,6 +11,7 @@
 
 %code requires {
     #include <string>
+    #include "ast.hpp"
     class Scanner;  
 }
 
@@ -23,52 +24,60 @@
 }
 
 %parse-param { Scanner* scanner }
+%parse-param { std::unique_ptr<Module>* result }
 
-// ---------------------- Semantic tokens ----------------------
 %token <std::string> IDENTIFIER STRING
-%token <int> NUMBER
+%token <std::string> NUMBER
 
-// ---------------------- Keywords ----------------------
 %left KW_OR
 %left KW_AND
 %right KW_NOT          // unary logical NOT
 %token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN KW_PASS KW_ELSE KW_ELIF KW_CLASS
 
-// ---------------------- Structure / punctuation ----------------------
 %token L_PAREN R_PAREN L_BRACKET R_BRACKET L_BRACE R_BRACE
 %token DOT COLON SEMICOLON COMMA
 %token INDENT DEDENT NEWLINE EOF
 
-// ---------------------- Operators ----------------------
-// Bitwise
 %left BIT_OR
 %left BIT_XOR
 %left BIT_AND
 %left LSHIFT RSHIFT
 %token BIT_NOT
 
-// Comparison
 %nonassoc EQ NE LT LTE GT GTE
 
-// Arithmetic
 %left PLUS MINUS
 %left MULTIPLY DIVIDE MODULO
-%right UMINUS         // unary minus
 
-// Assignment (right-associative)
 %right ASSIGN PLUS_ASSIGN MINUS_ASSIGN MULTIPLY_ASSIGN DIVIDE_ASSIGN MODULO_ASSIGN
 %right BIT_AND_ASSIGN BIT_OR_ASSIGN BIT_XOR_ASSIGN LSHIFT_ASSIGN RSHIFT_ASSIGN
 
+// Add types for your grammar rules:
+%type <std::unique_ptr<Expr>> expr
+%type <std::unique_ptr<Stmt>> stmt
+%type <std::vector<std::unique_ptr<Stmt>>> stmt_list
 
 
 %%
 input:
-      /* empty */
+      stmt_list EOF       { *result = std::make_unique<Module>(std::move($1));}
-    | input element
     ;
 
-element:
+stmt_list:
-      IDENTIFIER   { std::cout << "IDENTIFIER: " << $1 << "\n"; }
+      /* empty */         { $$ = std::vector<std::unique_ptr<Stmt>>(); }
+    | stmt_list stmt      { $1.push_back(std::move($2)); $$ = std::move($1); }
+    ;
+
+stmt:
+      expr                { $$ = std::make_unique<ExprStmt>(std::move($1)); }
+    ;
+
+expr:
+      NUMBER              { $$ = std::make_unique<Number>($1); }
+    | IDENTIFIER          { $$ = std::make_unique<Identifier>($1); }
+    | expr PLUS expr      { $$ = std::make_unique<BinOp>(
+                                BinaryOperator(BinaryOperator::ADD),
+                                std::move($1), std::move($3)); }
     ;
 
 %%

src/scanner.hpp

@@ -10,6 +10,8 @@
 
 class Scanner : public yyFlexLexer
 {
+    bool eof_emitted = false;
+
 public:
     IndentStack indent_stack;
     Scanner(std::istream &in) : yyFlexLexer(&in), yylval(nullptr) {}
@@ -23,6 +25,8 @@ public:
         this->yylval = lval;
         int token = yylex();
 
+        // std::cout << "lex: " << yy::parser::symbol_name(static_cast<yy::parser::symbol_kind_type>(token)) << '\n';
+
         if (token == yy::parser::token::TOK_EOF)
         {
             auto result = indent_stack.closeBlock();
@@ -33,6 +37,17 @@ public:
             }
         }
 
+        if (token != yy::parser::token::TOK_EOF)
+        {
+            return token;
+        }
+
+        if (eof_emitted)
+        {
+            return 0;
+        }
+
+        eof_emitted = true;
         return token;
     }
 

test/parser/1.in

@@ -0,0 +1 @@
+42

test/parser/1.out

@@ -0,0 +1,3 @@
+Module
+  ExprStmt
+    Number(42)

test/tester.sh

@@ -1,15 +1,30 @@
 #!/bin/bash
+cd "$(dirname "$0")"
 
-cd "$(dirname "$0")"  # Change to script directory
+echo "=== LEXER TESTS ==="
-
 for test in lexer/*.in; do
     name=$(basename "$test" .in)
-    echo "Testing $name..."
+    echo "Testing lexer/$name..."
     
     if ! ../build/py2rb --dump-tokens < "$test" | diff - "lexer/${name}.out"; then
-        echo "FAILED: $name"
+        echo "FAILED: lexer/$name"
         exit 1
     fi
 done
+echo "✓ All lexer tests passed"
 
+echo ""
+echo "=== PARSER TESTS ==="
+for test in parser/*.in; do
+    name=$(basename "$test" .in)
+    echo "Testing parser/$name..."
+    
+    if ! ../build/py2rb --dump-ast < "$test" | diff - "parser/${name}.out"; then
+        echo "FAILED: parser/$name"
+        exit 1
+    fi
+done
+echo "✓ All parser tests passed"
+
+echo ""
 echo "SUCCESS: All tests passed"