new interface

src/main.cpp

@@ -17,18 +17,18 @@ void dump_tokens(Scanner &scanner)
         int tok = scanner.lex(&lval);
         if (tok == Parser::token::TOK_EOF)
         {
-            std::cout << Parser::symbol_name(static_cast<kind>(tok));
+            // std::cout << Parser::symbol_name(static_cast<kind>(tok));
             break;
         }
 
-        std::cout << Parser::symbol_name(static_cast<kind>(tok));
+        // std::cout << Parser::symbol_name(static_cast<kind>(tok));
 
         if (tok == Parser::token::TOK_IDENTIFIER || tok == Parser::token::TOK_NUMBER || tok == Parser::token::TOK_STRING)
         {
-            std::cout << " " << lval.as<std::string>();
+            // std::cout << " " << lval.as<std::string>();
         }
 
-        std::cout << "\n";
+        // std::cout << "\n";
     }
 }
 
@@ -39,6 +39,8 @@ int main(int argc, char *argv[])
 
     bool token_dump = false;
     bool ast_dump = false;
+    bool write_ruby = true;
+    bool no_parsing = false;
 
     for (int i = 1; i < argc; ++i)
     {
@@ -56,10 +58,22 @@ int main(int argc, char *argv[])
         {
             ast_dump = true;
         }
+        else if (arg == "--no-output")
+        {
+            write_ruby = false;
+        }
+        else if (arg == "--no-parsing")
+        {
+            no_parsing = true;
+        }
     }
 
     Scanner scanner(*input_stream);
     if (token_dump)
+    {
+        scanner.print_results = true;
+    }
+    if (no_parsing)
     {
         dump_tokens(scanner);
         return 0;
@@ -72,9 +86,11 @@ int main(int argc, char *argv[])
     if (ast_dump)
     {
         std::cout << ast_root->dump();
-        return 0;
     }
 
-    RubyGenerator generator;
-    std::cout << generator.generate(*ast_root);
+    if (write_ruby)
+    {
+        RubyGenerator generator;
+        std::cout << generator.generate(*ast_root);
+    }
 }

src/scanner.hpp

@@ -14,6 +14,7 @@ class Scanner : public yyFlexLexer
     bool last_was_newline = true;
 
 public:
+    bool print_results = false;
     IndentStack indent_stack;
     Scanner(std::istream &in) : yyFlexLexer(&in), yylval(nullptr) {}
 
@@ -24,13 +25,15 @@ public:
 
         if (eof_emitted)
         {
-            // std::cout << "lex: 0\n";
             return 0;
         }
 
         if (indent_stack.popDedent())
         {
-            // std::cout << "lex: dedent\n";
+            if (print_results)
+            {
+                std::cout << "DEDENT\n";
+            }
             return yy::parser::token::TOK_DEDENT;
         }
 
@@ -39,18 +42,23 @@ public:
 
         if (token == yy::parser::token::TOK_EOF)
         {
-            // Emit synthetic NEWLINE if needed
             if (!last_was_newline)
             {
                 last_was_newline = true;
-                // std::cout << "lex: newline\n";
+                if (print_results)
+                {
+                    std::cout << "NEWLINE\n";
+                }
                 return yy::parser::token::TOK_NEWLINE;
             }
 
             auto result = indent_stack.closeBlock();
             if (result == IndentAction::Dedent)
             {
-                // std::cout << "lex: dedent\n";
+                if (print_results)
+                {
+                    std::cout << "DEDENT\n";
+                }
                 return yy::parser::token::TOK_DEDENT;
             }
 
@@ -59,7 +67,24 @@ public:
 
         last_was_newline = (token == yy::parser::token::TOK_NEWLINE);
 
-        // std::cout << "lex: " << Parser::symbol_name(static_cast<kind>(token)) << '\n';
+        if (print_results)
+        {
+            std::cout << Parser::symbol_name(static_cast<kind>(token));
+        }
+
+        if (token == Parser::token::TOK_IDENTIFIER || token == Parser::token::TOK_NUMBER || token == Parser::token::TOK_STRING)
+        {
+            if (print_results)
+            {
+                std::cout << " " << lval->as<std::string>();
+            }
+        }
+
+        if (print_results)
+        {
+            std::cout << "\n";
+        }
+
         return token;
     }
 

test/lexer/01.out

@@ -12,4 +12,4 @@ IDENTIFIER y
 MINUS
 IDENTIFIER z
 NEWLINE
-EOF
+EOF

test/lexer/02.out

@@ -28,4 +28,4 @@ R_PAREN
 NEWLINE
 DEDENT
 DEDENT
-EOF
+EOF

test/lexer/03.out

@@ -70,4 +70,4 @@ KW_RETURN
 IDENTIFIER result
 NEWLINE
 DEDENT
-EOF
+EOF

test/lexer/04.out

@@ -31,4 +31,4 @@ COLON
 DOT
 SEMICOLON
 NEWLINE
-EOF
+EOF

test/lexer/05.out

@@ -10,4 +10,4 @@ NEWLINE
 STRING 'test'
 STRING "escaped\"quote"
 NEWLINE
-EOF
+EOF

test/lexer/06.out

@@ -48,4 +48,4 @@ COMMA
 NUMBER 15
 R_PAREN
 NEWLINE
-EOF
+EOF

test/lexer/07.out

@@ -18,4 +18,4 @@ IDENTIFIER very_long_variable_name_123
 ASSIGN
 STRING "test"
 NEWLINE
-EOF
+EOF

test/lexer/08.out

@@ -16,4 +16,4 @@ NEWLINE
 KW_NOT
 IDENTIFIER flag
 NEWLINE
-EOF
+EOF

test/lexer/09.out

@@ -10,4 +10,4 @@ INDENT
 KW_PASS
 NEWLINE
 DEDENT
-EOF
+EOF

test/lexer/10.out

@@ -10,4 +10,4 @@ IDENTIFIER e
 RSHIFT
 IDENTIFIER f
 NEWLINE
-EOF
+EOF

test/lexer/11.out

@@ -38,4 +38,4 @@ IDENTIFIER g
 RSHIFT_ASSIGN
 NUMBER 10
 NEWLINE
-EOF
+EOF

test/lexer/12.out

@@ -16,4 +16,4 @@ L_BRACKET
 IDENTIFIER index
 R_BRACKET
 NEWLINE
-EOF
+EOF

test/lexer/13.out

@@ -38,4 +38,4 @@ COMMA
 NUMBER 3
 R_BRACE
 NEWLINE
-EOF
+EOF

test/lexer/14.out

@@ -4,4 +4,4 @@ STRING 'quote\'test'
 NEWLINE
 STRING "unicode ☃ snowman"
 NEWLINE
-EOF
+EOF

test/lexer/15.out

@@ -13,4 +13,4 @@ INDENT
 KW_PASS
 NEWLINE
 DEDENT
-EOF
+EOF

test/lexer/16.out

@@ -12,4 +12,4 @@ NUMBER 5
 BIT_XOR
 NUMBER 6
 NEWLINE
-EOF
+EOF

test/lexer/17.out

@@ -10,4 +10,4 @@ IDENTIFIER s3
 ASSIGN
 STRING "mixed 'quotes' \"inside\""
 NEWLINE
-EOF
+EOF

test/lexer/18.out

@@ -26,4 +26,4 @@ NUMBER 7
 R_BRACE
 R_BRACKET
 NEWLINE
-EOF
+EOF

test/lexer/19.out

@@ -167,4 +167,4 @@ COMMA
 NUMBER 15
 R_PAREN
 NEWLINE
-EOF
+EOF

test/tester.sh

@@ -6,7 +6,7 @@ for test in lexer/*.in; do
     name=$(basename "$test" .in)
     echo "Testing lexer/$name..."
     
-    if ! ../build/py2rb --dump-tokens < "$test" | diff - "lexer/${name}.out"; then
+    if ! ../build/py2rb --dump-tokens --no-output --no-parsing < "$test" | diff - "lexer/${name}.out"; then
         echo "FAILED: lexer/$name"
         exit 1
     fi
@@ -19,7 +19,7 @@ for test in parser/*.in; do
     name=$(basename "$test" .in)
     echo "Testing parser/$name..."
     
-    if ! ../build/py2rb --dump-ast < "$test" | diff - "parser/${name}.out"; then
+    if ! ../build/py2rb --dump-ast --no-output < "$test" | diff - "parser/${name}.out"; then
         echo "FAILED: parser/$name"
         exit 1
     fi