From c76b84682a417b8ac22afc1414e03e8ac6f5bda1 Mon Sep 17 00:00:00 2001 From: bronku Date: Thu, 27 Nov 2025 15:45:10 +0100 Subject: [PATCH] parser start --- makefile | 4 +- src/ast.hpp | 7 ++ src/ast/base.hpp | 15 +++ src/ast/expressions.hpp | 145 +++++++++++++++++++++++++++++ src/ast/literals.hpp | 39 ++++++++ src/ast/operators.hpp | 105 +++++++++++++++++++++ src/ast/statements.hpp | 192 +++++++++++++++++++++++++++++++++++++++ src/ast/string_utils.hpp | 7 ++ src/main.cpp | 17 +++- src/parser.y | 37 +++++--- src/scanner.hpp | 15 +++ test/parser/1.in | 1 + test/parser/1.out | 3 + test/tester.sh | 23 ++++- 14 files changed, 589 insertions(+), 21 deletions(-) create mode 100644 src/ast.hpp create mode 100644 src/ast/base.hpp create mode 100644 src/ast/expressions.hpp create mode 100644 src/ast/literals.hpp create mode 100644 src/ast/operators.hpp create mode 100644 src/ast/statements.hpp create mode 100644 src/ast/string_utils.hpp create mode 100644 test/parser/1.in create mode 100644 test/parser/1.out diff --git a/makefile b/makefile index 7d47ebf..6ed9765 100644 --- a/makefile +++ b/makefile @@ -11,7 +11,7 @@ $(TARGET): build/parser.o build/lexer.o build/main.o build/parser.cpp build/parser.hpp: src/parser.y | build $(BISON) -d -o build/parser.cpp src/parser.y -build/lexer.cpp build/lexer.hpp: src/lexer.l build/parser.hpp src/scanner.hpp src/indent_stack.hpp| build +build/lexer.cpp build/lexer.hpp: src/lexer.l build/parser.hpp src/scanner.hpp src/indent_stack.hpp build $(FLEX) --header-file=build/lexer.hpp -o build/lexer.cpp src/lexer.l build/%.o: build/%.cpp @@ -20,6 +20,8 @@ build/%.o: build/%.cpp build/%.o: src/%.cpp | build $(CXX) -c $(ARGS) $< -o $@ +src/ast.hpp: src/ast/base.hpp src/ast/expressions.hpp src/ast/literals.hpp src/ast/operators.hpp src/ast/statements.hpp src/ast/string_utils.hpp + build: mkdir -p build diff --git a/src/ast.hpp b/src/ast.hpp new file mode 100644 index 0000000..93d9720 --- /dev/null +++ b/src/ast.hpp @@ -0,0 +1,7 @@ +#pragma once +#include "ast/string_utils.hpp" +#include "ast/base.hpp" +#include "ast/expressions.hpp" +#include "ast/literals.hpp" +#include "ast/operators.hpp" +#include "ast/statements.hpp" \ No newline at end of file diff --git a/src/ast/base.hpp b/src/ast/base.hpp new file mode 100644 index 0000000..140ce34 --- /dev/null +++ b/src/ast/base.hpp @@ -0,0 +1,15 @@ +#pragma once +#include + +struct Node +{ + virtual ~Node() = default; + virtual std::string dump(int indent = 0) const = 0; +}; + +struct Expr : Node +{ +}; +struct Stmt : Node +{ +}; \ No newline at end of file diff --git a/src/ast/expressions.hpp b/src/ast/expressions.hpp new file mode 100644 index 0000000..4284dc1 --- /dev/null +++ b/src/ast/expressions.hpp @@ -0,0 +1,145 @@ +#pragma once +#include +#include "base.hpp" +#include "operators.hpp" +#include "literals.hpp" + +template +using ptr = std::unique_ptr; + +struct BinOp : Expr +{ + BinaryOperator op; + ptr left; + ptr right; + + BinOp(BinaryOperator o, ptr l, ptr r) + : op(o), left(std::move(l)), right(std::move(r)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "BinOp(" << op.symbol() << ")\n"; + oss << left->dump(indent + 1) << "\n"; + oss << right->dump(indent + 1); + return oss.str(); + } +}; + +struct UnaryOpExpr : Expr +{ + UnaryOperator op; + ptr operand; + + UnaryOpExpr(UnaryOperator o, ptr expr) + : op(o), operand(std::move(expr)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "UnaryOp(" << op.symbol() << ")\n"; + oss << operand->dump(indent + 1); + return oss.str(); + } +}; + +struct Call : Expr +{ + ptr func; + std::vector> args; + + Call(ptr f, std::vector> a) + : func(std::move(f)), args(std::move(a)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Call\n"; + oss << func->dump(indent + 1); + for (const auto &arg : args) + { + oss << "\n" + << arg->dump(indent + 1); + } + return oss.str(); + } +}; + +struct Attribute : Expr +{ + ptr value; + std::string attr; + + Attribute(ptr v, std::string a) + : value(std::move(v)), attr(a) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Attribute(." << attr << ")\n"; + oss << value->dump(indent + 1); + return oss.str(); + } +}; + +struct Subscript : Expr +{ + ptr value; + ptr index; + + Subscript(ptr v, ptr i) + : value(std::move(v)), index(std::move(i)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Subscript\n"; + oss << value->dump(indent + 1) << "\n"; + oss << index->dump(indent + 1); + return oss.str(); + } +}; + +struct List : Expr +{ + std::vector> elements; + + List(std::vector> elems) + : elements(std::move(elems)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "List"; + for (const auto &elem : elements) + { + oss << "\n" + << elem->dump(indent + 1); + } + return oss.str(); + } +}; + +struct Dict : Expr +{ + std::vector, ptr>> pairs; + + Dict(std::vector, ptr>> p) + : pairs(std::move(p)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Dict"; + for (const auto &[key, val] : pairs) + { + oss << "\n" + << indent_str(indent + 1) << "Pair"; + oss << "\n" + << key->dump(indent + 2); + oss << "\n" + << val->dump(indent + 2); + } + return oss.str(); + } +}; \ No newline at end of file diff --git a/src/ast/literals.hpp b/src/ast/literals.hpp new file mode 100644 index 0000000..92b0e9a --- /dev/null +++ b/src/ast/literals.hpp @@ -0,0 +1,39 @@ +#pragma once +#include "base.hpp" +#include "string_utils.hpp" + +struct Identifier : Expr +{ + std::string name; + + Identifier(std::string n) : name(n) {} + + std::string dump(int indent = 0) const override + { + return indent_str(indent) + "Identifier(" + name + ")"; + } +}; + +struct Number : Expr +{ + std::string value; + + Number(std::string v) : value(v) {} + + std::string dump(int indent = 0) const override + { + return indent_str(indent) + "Number(" + value + ")"; + } +}; + +struct String : Expr +{ + std::string value; + + String(std::string v) : value(v) {} + + std::string dump(int indent = 0) const override + { + return indent_str(indent) + "String(\"" + value + "\")"; + } +}; \ No newline at end of file diff --git a/src/ast/operators.hpp b/src/ast/operators.hpp new file mode 100644 index 0000000..ae7f553 --- /dev/null +++ b/src/ast/operators.hpp @@ -0,0 +1,105 @@ +#pragma once +#include + +struct BinaryOperator +{ + enum Kind + { + ADD, + SUB, + MUL, + DIV, + MOD, + EQ, + NE, + LT, + LTE, + GT, + GTE, + AND, + OR, + BIT_AND, + BIT_OR, + BIT_XOR, + LSHIFT, + RSHIFT + }; + + Kind kind; + + BinaryOperator(Kind k) : kind(k) {} + + std::string symbol() const + { + switch (kind) + { + case ADD: + return "+"; + case SUB: + return "-"; + case MUL: + return "*"; + case DIV: + return "/"; + case MOD: + return "%"; + case EQ: + return "=="; + case NE: + return "!="; + case LT: + return "<"; + case LTE: + return "<="; + case GT: + return ">"; + case GTE: + return ">="; + case AND: + return "and"; + case OR: + return "or"; + case BIT_AND: + return "&"; + case BIT_OR: + return "|"; + case BIT_XOR: + return "^"; + case LSHIFT: + return "<<"; + case RSHIFT: + return ">>"; + default: + return "?"; + } + } +}; + +struct UnaryOperator +{ + enum Kind + { + NOT, + NEG, + BIT_NOT + }; + + Kind kind; + + UnaryOperator(Kind k) : kind(k) {} + + std::string symbol() const + { + switch (kind) + { + case NOT: + return "not"; + case NEG: + return "-"; + case BIT_NOT: + return "~"; + default: + return "?"; + } + } +}; \ No newline at end of file diff --git a/src/ast/statements.hpp b/src/ast/statements.hpp new file mode 100644 index 0000000..43ab8d1 --- /dev/null +++ b/src/ast/statements.hpp @@ -0,0 +1,192 @@ +#pragma once +#include +#include +#include "string_utils.hpp" +#include "base.hpp" + +template +using ptr = std::unique_ptr; + +struct Assign : Stmt +{ + ptr target; + ptr value; + + Assign(ptr t, ptr v) + : target(std::move(t)), value(std::move(v)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Assign\n"; + oss << target->dump(indent + 1) << "\n"; + oss << value->dump(indent + 1); + return oss.str(); + } +}; + +struct ExprStmt : Stmt +{ + ptr expr; + + ExprStmt(ptr e) : expr(std::move(e)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "ExprStmt\n"; + oss << expr->dump(indent + 1); + return oss.str(); + } +}; + +struct Return : Stmt +{ + ptr value; + + Return(ptr v) : value(std::move(v)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "Return\n"; + if (value) + { + oss << value->dump(indent + 1); + } + return oss.str(); + } +}; + +struct Pass : Stmt +{ + std::string dump(int indent = 0) const override + { + return indent_str(indent) + "Pass"; + } +}; + +struct If : Stmt +{ + ptr condition; + std::vector> then_body; + std::vector> else_body; + + If(ptr cond, std::vector> then_b, std::vector> else_b) + : condition(std::move(cond)), + then_body(std::move(then_b)), + else_body(std::move(else_b)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "If\n"; + oss << condition->dump(indent + 1); + for (const auto &stmt : then_body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + if (!else_body.empty()) + { + oss << "\n" + << indent_str(indent + 1) << "Else"; + for (const auto &stmt : else_body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + } + return oss.str(); + } +}; + +struct For : Stmt +{ + std::string target; + ptr iter; + std::vector> body; + + For(std::string t, ptr i, std::vector> b) + : target(t), iter(std::move(i)), body(std::move(b)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "For(" << target << ")\n"; + oss << iter->dump(indent + 1); + for (const auto &stmt : body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + return oss.str(); + } +}; + +struct FunctionDef : Stmt +{ + std::string name; + std::vector params; + std::vector> body; + + FunctionDef(std::string n, std::vector p, std::vector> b) + : name(n), params(std::move(p)), body(std::move(b)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "FunctionDef(" << name << ")"; + for (const auto ¶m : params) + { + oss << "\n" + << indent_str(indent + 1) << "Param(" << param << ")"; + } + for (const auto &stmt : body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + return oss.str(); + } +}; + +struct ClassDef : Stmt +{ + std::string name; + std::vector> body; + + ClassDef(std::string n, std::vector> b) + : name(n), body(std::move(b)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << indent_str(indent) << "ClassDef(" << name << ")"; + for (const auto &stmt : body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + return oss.str(); + } +}; + +struct Module : Node +{ + std::vector> body; + + Module(std::vector> b) : body(std::move(b)) {} + + std::string dump(int indent = 0) const override + { + std::ostringstream oss; + oss << "Module"; + for (const auto &stmt : body) + { + oss << "\n" + << stmt->dump(indent + 1); + } + return oss.str(); + } +}; \ No newline at end of file diff --git a/src/ast/string_utils.hpp b/src/ast/string_utils.hpp new file mode 100644 index 0000000..a2168ff --- /dev/null +++ b/src/ast/string_utils.hpp @@ -0,0 +1,7 @@ +#pragma once +#include + +inline std::string indent_str(int level) +{ + return std::string(level * 2, ' '); +} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index bae4349..1b47df6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,6 @@ #include "parser.hpp" #include "scanner.hpp" +#include "ast.hpp" #include #include @@ -36,6 +37,7 @@ int main(int argc, char *argv[]) std::ifstream file_stream; bool token_dump = false; + bool ast_dump = false; for (int i = 1; i < argc; ++i) { @@ -49,6 +51,10 @@ int main(int argc, char *argv[]) { token_dump = true; } + else if (arg == "--dump-ast") + { + ast_dump = true; + } } Scanner scanner(*input_stream); @@ -57,6 +63,13 @@ int main(int argc, char *argv[]) dump_tokens(scanner); return 0; } - yy::parser parser(&scanner); - return parser.parse(); + + std::unique_ptr ast_root; + yy::parser parser(&scanner, &ast_root); + parser.parse(); + + if (ast_dump) + { + std::cout << ast_root->dump(); + } } \ No newline at end of file diff --git a/src/parser.y b/src/parser.y index cfcb691..828505a 100644 --- a/src/parser.y +++ b/src/parser.y @@ -11,6 +11,7 @@ %code requires { #include + #include "ast.hpp" class Scanner; } @@ -23,52 +24,60 @@ } %parse-param { Scanner* scanner } +%parse-param { std::unique_ptr* result } -// ---------------------- Semantic tokens ---------------------- %token IDENTIFIER STRING -%token NUMBER +%token NUMBER -// ---------------------- Keywords ---------------------- %left KW_OR %left KW_AND %right KW_NOT // unary logical NOT %token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN KW_PASS KW_ELSE KW_ELIF KW_CLASS -// ---------------------- Structure / punctuation ---------------------- %token L_PAREN R_PAREN L_BRACKET R_BRACKET L_BRACE R_BRACE %token DOT COLON SEMICOLON COMMA %token INDENT DEDENT NEWLINE EOF -// ---------------------- Operators ---------------------- -// Bitwise %left BIT_OR %left BIT_XOR %left BIT_AND %left LSHIFT RSHIFT %token BIT_NOT -// Comparison %nonassoc EQ NE LT LTE GT GTE -// Arithmetic %left PLUS MINUS %left MULTIPLY DIVIDE MODULO -%right UMINUS // unary minus -// Assignment (right-associative) %right ASSIGN PLUS_ASSIGN MINUS_ASSIGN MULTIPLY_ASSIGN DIVIDE_ASSIGN MODULO_ASSIGN %right BIT_AND_ASSIGN BIT_OR_ASSIGN BIT_XOR_ASSIGN LSHIFT_ASSIGN RSHIFT_ASSIGN +// Add types for your grammar rules: +%type > expr +%type > stmt +%type >> stmt_list %% input: - /* empty */ - | input element + stmt_list EOF { *result = std::make_unique(std::move($1));} ; -element: - IDENTIFIER { std::cout << "IDENTIFIER: " << $1 << "\n"; } +stmt_list: + /* empty */ { $$ = std::vector>(); } + | stmt_list stmt { $1.push_back(std::move($2)); $$ = std::move($1); } + ; + +stmt: + expr { $$ = std::make_unique(std::move($1)); } + ; + +expr: + NUMBER { $$ = std::make_unique($1); } + | IDENTIFIER { $$ = std::make_unique($1); } + | expr PLUS expr { $$ = std::make_unique( + BinaryOperator(BinaryOperator::ADD), + std::move($1), std::move($3)); } ; %% diff --git a/src/scanner.hpp b/src/scanner.hpp index 72092e3..b914320 100644 --- a/src/scanner.hpp +++ b/src/scanner.hpp @@ -10,6 +10,8 @@ class Scanner : public yyFlexLexer { + bool eof_emitted = false; + public: IndentStack indent_stack; Scanner(std::istream &in) : yyFlexLexer(&in), yylval(nullptr) {} @@ -23,6 +25,8 @@ public: this->yylval = lval; int token = yylex(); + // std::cout << "lex: " << yy::parser::symbol_name(static_cast(token)) << '\n'; + if (token == yy::parser::token::TOK_EOF) { auto result = indent_stack.closeBlock(); @@ -33,6 +37,17 @@ public: } } + if (token != yy::parser::token::TOK_EOF) + { + return token; + } + + if (eof_emitted) + { + return 0; + } + + eof_emitted = true; return token; } diff --git a/test/parser/1.in b/test/parser/1.in new file mode 100644 index 0000000..f70d7bb --- /dev/null +++ b/test/parser/1.in @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/test/parser/1.out b/test/parser/1.out new file mode 100644 index 0000000..95170bc --- /dev/null +++ b/test/parser/1.out @@ -0,0 +1,3 @@ +Module + ExprStmt + Number(42) \ No newline at end of file diff --git a/test/tester.sh b/test/tester.sh index 1e4ebc2..7ef7bf2 100755 --- a/test/tester.sh +++ b/test/tester.sh @@ -1,15 +1,30 @@ #!/bin/bash +cd "$(dirname "$0")" -cd "$(dirname "$0")" # Change to script directory - +echo "=== LEXER TESTS ===" for test in lexer/*.in; do name=$(basename "$test" .in) - echo "Testing $name..." + echo "Testing lexer/$name..." if ! ../build/py2rb --dump-tokens < "$test" | diff - "lexer/${name}.out"; then - echo "FAILED: $name" + echo "FAILED: lexer/$name" exit 1 fi done +echo "✓ All lexer tests passed" +echo "" +echo "=== PARSER TESTS ===" +for test in parser/*.in; do + name=$(basename "$test" .in) + echo "Testing parser/$name..." + + if ! ../build/py2rb --dump-ast < "$test" | diff - "parser/${name}.out"; then + echo "FAILED: parser/$name" + exit 1 + fi +done +echo "✓ All parser tests passed" + +echo "" echo "SUCCESS: All tests passed" \ No newline at end of file