Add pytest test suite for lexer, parser and integration

tests/conftest.py

+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import pytest
+from bootstrap.lexer import Lexer
+from bootstrap.parser import Parser
+from bootstrap.codegen import CodeGenerator
+
+
+@pytest.fixture
+def lex():
+    def _lex(source: str):
+        lexer = Lexer(source)
+        return lexer.tokenize()
+    return _lex
+
+
+@pytest.fixture
+def parse():
+    def _parse(source: str):
+        lexer = Lexer(source)
+        tokens = lexer.tokenize()
+        parser = Parser(tokens)
+        return parser.parse()
+    return _parse
+
+
+@pytest.fixture
+def compile_ct():
+    def _compile(source: str) -> str:
+        lexer = Lexer(source)
+        tokens = lexer.tokenize()
+        parser = Parser(tokens)
+        ast = parser.parse()
+        gen = CodeGenerator()
+        return gen.generate(ast)
+    return _compile

tests/test_integration.py

+import subprocess
+import tempfile
+import os
+import pytest
+
+
+def run_ct(source: str) -> tuple[int, str, str]:
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.ct', delete=False) as f:
+        f.write(source)
+        f.flush()
+        ct_file = f.name
+
+    try:
+        result = subprocess.run(
+            ['python3', 'content', 'run', ct_file],
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        return result.returncode, result.stdout, result.stderr
+    finally:
+        os.unlink(ct_file)
+
+
+def compile_ct(source: str) -> tuple[int, str, str]:
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.ct', delete=False) as f:
+        f.write(source)
+        f.flush()
+        ct_file = f.name
+
+    sh_file = ct_file.replace('.ct', '.sh')
+
+    try:
+        result = subprocess.run(
+            ['python3', 'content', 'build', ct_file, '-o', sh_file],
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        compiled_output = ""
+        if os.path.exists(sh_file):
+            with open(sh_file, 'r') as sf:
+                compiled_output = sf.read()
+            os.unlink(sh_file)
+        return result.returncode, compiled_output, result.stderr
+    finally:
+        os.unlink(ct_file)
+
+
+class TestPrint:
+    def test_print_string(self):
+        code, stdout, _ = run_ct('print ("hello")')
+        assert code == 0
+        assert "hello" in stdout
+
+    def test_print_number(self):
+        code, stdout, _ = run_ct('print (42)')
+        assert code == 0
+        assert "42" in stdout
+
+    def test_print_interpolation(self):
+        code, stdout, _ = run_ct('name = "World"\nprint ("Hello, {name}!")')
+        assert code == 0
+        assert "Hello, World!" in stdout
+
+
+class TestVariables:
+    def test_variable_assignment(self):
+        code, stdout, _ = run_ct('x = 10\nprint (x)')
+        assert code == 0
+        assert "10" in stdout
+
+    def test_variable_reassignment(self):
+        code, stdout, _ = run_ct('x = 10\nx = 20\nprint (x)')
+        assert code == 0
+        assert "20" in stdout
+
+
+class TestArithmetic:
+    def test_addition(self):
+        code, stdout, _ = run_ct('print (1 + 2)')
+        assert code == 0
+        assert "3" in stdout
+
+    def test_subtraction(self):
+        code, stdout, _ = run_ct('print (10 - 3)')
+        assert code == 0
+        assert "7" in stdout
+
+    def test_multiplication(self):
+        code, stdout, _ = run_ct('print (4 * 5)')
+        assert code == 0
+        assert "20" in stdout
+
+    def test_division(self):
+        code, stdout, _ = run_ct('print (10 / 2)')
+        assert code == 0
+        assert "5" in stdout
+
+    def test_modulo(self):
+        code, stdout, _ = run_ct('print (10 % 3)')
+        assert code == 0
+        assert "1" in stdout
+
+
+class TestFunctions:
+    def test_function_call(self):
+        code, stdout, _ = run_ct('''
+func greet () {
+    print ("hello")
+}
+greet ()
+''')
+        assert code == 0
+        assert "hello" in stdout
+
+    def test_function_with_param(self):
+        code, stdout, _ = run_ct('''
+func greet (name) {
+    print ("Hello, {name}")
+}
+greet ("Alice")
+''')
+        assert code == 0
+        assert "Hello, Alice" in stdout
+
+    def test_function_return(self):
+        code, stdout, _ = run_ct('''
+func add (a, b) {
+    return a + b
+}
+result = add (3, 4)
+print (result)
+''')
+        assert code == 0
+        assert "7" in stdout
+
+    def test_function_default_param(self):
+        code, stdout, _ = run_ct('''
+func greet (name = "World") {
+    print ("Hello, {name}")
+}
+greet ()
+''')
+        assert code == 0
+        assert "Hello, World" in stdout
+
+
+class TestControlFlow:
+    def test_if_true(self):
+        code, stdout, _ = run_ct('''
+x = 10
+if x > 5 {
+    print ("big")
+}
+''')
+        assert code == 0
+        assert "big" in stdout
+
+    def test_if_false(self):
+        code, stdout, _ = run_ct('''
+x = 3
+if x > 5 {
+    print ("big")
+} else {
+    print ("small")
+}
+''')
+        assert code == 0
+        assert "small" in stdout
+
+    def test_while_loop(self):
+        code, stdout, _ = run_ct('''
+i = 0
+while i < 3 {
+    print (i)
+    i += 1
+}
+''')
+        assert code == 0
+        assert "0" in stdout
+        assert "1" in stdout
+        assert "2" in stdout
+
+    def test_foreach_range(self):
+        code, stdout, _ = run_ct('''
+foreach i in range (3) {
+    print (i)
+}
+''')
+        assert code == 0
+        assert "0" in stdout
+        assert "1" in stdout
+        assert "2" in stdout
+
+
+class TestArrays:
+    def test_array_create(self):
+        code, stdout, _ = run_ct('''
+arr = [1, 2, 3]
+print (arr[0])
+''')
+        assert code == 0
+        assert "1" in stdout
+
+    def test_array_push(self):
+        code, stdout, _ = run_ct('''
+arr = [1, 2]
+arr.push (3)
+print (arr.len ())
+''')
+        assert code == 0
+        assert "3" in stdout
+
+
+class TestClasses:
+    def test_class_instantiation(self):
+        code, stdout, _ = run_ct('''
+class Counter {
+    value = 0
+
+    func inc () {
+        this.value += 1
+    }
+
+    func get () {
+        return this.value
+    }
+}
+
+c = Counter ()
+c.inc ()
+c.inc ()
+print (c.get ())
+''')
+        assert code == 0
+        assert "2" in stdout
+
+    def test_class_constructor(self):
+        code, stdout, _ = run_ct('''
+class Person {
+    name = ""
+
+    construct (name) {
+        this.name = name
+    }
+
+    func greet () {
+        print ("Hello, {this.name}")
+    }
+}
+
+p = Person ("Alice")
+p.greet ()
+''')
+        assert code == 0
+        assert "Hello, Alice" in stdout
+
+
+class TestLambdas:
+    def test_simple_lambda(self):
+        code, stdout, _ = run_ct('''
+double = x => x * 2
+print (double (5))
+''')
+        assert code == 0
+        assert "10" in stdout
+
+    def test_multi_param_lambda(self):
+        code, stdout, _ = run_ct('''
+add = (a, b) => a + b
+print (add (3, 4))
+''')
+        assert code == 0
+        assert "7" in stdout
+
+
+class TestStringMethods:
+    def test_upper(self):
+        code, stdout, _ = run_ct('''
+s = "hello"
+print (s.upper ())
+''')
+        assert code == 0
+        assert "HELLO" in stdout
+
+    def test_lower(self):
+        code, stdout, _ = run_ct('''
+s = "HELLO"
+print (s.lower ())
+''')
+        assert code == 0
+        assert "hello" in stdout
+
+    def test_contains(self):
+        code, stdout, _ = run_ct('''
+s = "hello world"
+if s.contains ("world") {
+    print ("yes")
+}
+''')
+        assert code == 0
+        assert "yes" in stdout
+
+
+class TestExceptionHandling:
+    def test_try_except(self):
+        code, stdout, _ = run_ct('''
+try {
+    throw "oops"
+} except {
+    print ("caught")
+}
+''')
+        assert code == 0
+        assert "caught" in stdout
+
+    def test_defer(self):
+        code, stdout, _ = run_ct('''
+func test () {
+    defer print ("cleanup")
+    print ("work")
+}
+test ()
+''')
+        assert code == 0
+        lines = stdout.strip().split('\n')
+        assert "work" in lines[0]
+        assert "cleanup" in lines[1]
+
+
+class TestDecorators:
+    def test_log_decorator(self):
+        code, stdout, stderr = run_ct('''
+@log
+func hello () {
+    print ("hello")
+}
+hello ()
+''')
+        assert code == 0
+        assert "hello" in stdout
+
+    def test_retry_decorator(self):
+        code, stdout, _ = run_ct('''
+attempt = 0
+
+@retry (attempts=3)
+func flaky () {
+    attempt += 1
+    if attempt < 3 {
+        throw "fail"
+    }
+    print ("success")
+}
+
+try {
+    flaky ()
+} except {
+    print ("all failed")
+}
+''')
+        assert code == 0
+
+
+class TestStdlib:
+    def test_len_string(self):
+        code, stdout, _ = run_ct('''
+s = "hello"
+print (len (s))
+''')
+        assert code == 0
+        assert "5" in stdout
+
+    def test_math_abs(self):
+        code, stdout, _ = run_ct('''
+print (math.abs (-5))
+''')
+        assert code == 0
+        assert "5" in stdout
+
+    def test_range(self):
+        code, stdout, _ = run_ct('''
+foreach i in range (1, 4) {
+    print (i)
+}
+''')
+        assert code == 0
+        assert "1" in stdout
+        assert "2" in stdout
+        assert "3" in stdout
+
+
+class TestCompileOnly:
+    def test_compile_hello(self):
+        code, stdout, stderr = compile_ct('print ("hello")')
+        assert code == 0
+        assert "#!/usr/bin/env bash" in stdout
+
+    def test_compile_function(self):
+        code, stdout, stderr = compile_ct('''
+func greet () {
+    print ("hi")
+}
+greet ()
+''')
+        assert code == 0
+        assert "greet" in stdout

tests/test_lexer.py

+import pytest
+from bootstrap.lexer import Lexer
+from bootstrap.tokens import TokenType
+
+
+class TestLexerBasics:
+    def test_empty_source(self, lex):
+        tokens = lex("")
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.EOF
+
+    def test_whitespace_only(self, lex):
+        tokens = lex("   \t  ")
+        assert len(tokens) == 1
+        assert tokens[0].type == TokenType.EOF
+
+    def test_newlines(self, lex):
+        tokens = lex("\n\n")
+        assert tokens[0].type == TokenType.NEWLINE
+        assert tokens[1].type == TokenType.NEWLINE
+        assert tokens[2].type == TokenType.EOF
+
+
+class TestLexerNumbers:
+    def test_integer(self, lex):
+        tokens = lex("42")
+        assert tokens[0].type == TokenType.INTEGER
+        assert tokens[0].value == 42
+
+    def test_float(self, lex):
+        tokens = lex("3.14")
+        assert tokens[0].type == TokenType.FLOAT
+        assert tokens[0].value == 3.14
+
+    def test_multiple_numbers(self, lex):
+        tokens = lex("1 2 3")
+        assert tokens[0].value == 1
+        assert tokens[1].value == 2
+        assert tokens[2].value == 3
+
+
+class TestLexerStrings:
+    def test_simple_string(self, lex):
+        tokens = lex('"hello"')
+        assert tokens[0].type == TokenType.STRING
+        assert tokens[0].value == "hello"
+
+    def test_single_quotes(self, lex):
+        tokens = lex("'world'")
+        assert tokens[0].type == TokenType.STRING
+        assert tokens[0].value == "world"
+
+    def test_escape_newline(self, lex):
+        tokens = lex(r'"line1\nline2"')
+        assert tokens[0].value == "line1\nline2"
+
+    def test_escape_tab(self, lex):
+        tokens = lex(r'"col1\tcol2"')
+        assert tokens[0].value == "col1\tcol2"
+
+    def test_escape_braces(self, lex):
+        tokens = lex(r'"literal \{ brace \}"')
+        assert "\x00LBRACE\x00" in tokens[0].value
+        assert "\x00RBRACE\x00" in tokens[0].value
+
+
+class TestLexerIdentifiers:
+    def test_identifier(self, lex):
+        tokens = lex("myVar")
+        assert tokens[0].type == TokenType.IDENTIFIER
+        assert tokens[0].value == "myVar"
+
+    def test_identifier_with_underscore(self, lex):
+        tokens = lex("my_var_123")
+        assert tokens[0].type == TokenType.IDENTIFIER
+        assert tokens[0].value == "my_var_123"
+
+
+class TestLexerKeywords:
+    @pytest.mark.parametrize("keyword,expected", [
+        ("func", TokenType.FUNC),
+        ("class", TokenType.CLASS),
+        ("construct", TokenType.CONSTRUCT),
+        ("this", TokenType.THIS),
+        ("base", TokenType.BASE),
+        ("return", TokenType.RETURN),
+        ("if", TokenType.IF),
+        ("else", TokenType.ELSE),
+        ("foreach", TokenType.FOREACH),
+        ("for", TokenType.FOR),
+        ("in", TokenType.IN),
+        ("while", TokenType.WHILE),
+        ("break", TokenType.BREAK),
+        ("continue", TokenType.CONTINUE),
+        ("import", TokenType.IMPORT),
+        ("try", TokenType.TRY),
+        ("except", TokenType.EXCEPT),
+        ("finally", TokenType.FINALLY),
+        ("throw", TokenType.THROW),
+        ("defer", TokenType.DEFER),
+        ("range", TokenType.RANGE),
+        ("when", TokenType.WHEN),
+        ("new", TokenType.NEW),
+        ("true", TokenType.TRUE),
+        ("false", TokenType.FALSE),
+        ("nil", TokenType.NIL),
+    ])
+    def test_keyword(self, lex, keyword, expected):
+        tokens = lex(keyword)
+        assert tokens[0].type == expected
+
+
+class TestLexerOperators:
+    @pytest.mark.parametrize("op,expected", [
+        ("+", TokenType.PLUS),
+        ("-", TokenType.MINUS),
+        ("*", TokenType.STAR),
+        ("/", TokenType.SLASH),
+        ("%", TokenType.PERCENT),
+        ("=", TokenType.ASSIGN),
+        ("<", TokenType.LT),
+        (">", TokenType.GT),
+        ("!", TokenType.NOT),
+        (".", TokenType.DOT),
+        (":", TokenType.COLON),
+        ("(", TokenType.LPAREN),
+        (")", TokenType.RPAREN),
+        ("{", TokenType.LBRACE),
+        ("}", TokenType.RBRACE),
+        ("[", TokenType.LBRACKET),
+        ("]", TokenType.RBRACKET),
+        (",", TokenType.COMMA),
+        ("@", TokenType.AT),
+        ("|", TokenType.PIPE),
+    ])
+    def test_single_char_operator(self, lex, op, expected):
+        tokens = lex(op)
+        assert tokens[0].type == expected
+
+    @pytest.mark.parametrize("op,expected", [
+        ("==", TokenType.EQ),
+        ("!=", TokenType.NEQ),
+        ("<=", TokenType.LTE),
+        (">=", TokenType.GTE),
+        ("&&", TokenType.AND),
+        ("||", TokenType.OR),
+        ("=>", TokenType.ARROW),
+        ("+=", TokenType.PLUS_ASSIGN),
+        ("-=", TokenType.MINUS_ASSIGN),
+        ("*=", TokenType.STAR_ASSIGN),
+        ("/=", TokenType.SLASH_ASSIGN),
+        ("..", TokenType.DOTDOT),
+        ("...", TokenType.DOTDOTDOT),
+    ])
+    def test_multi_char_operator(self, lex, op, expected):
+        tokens = lex(op)
+        assert tokens[0].type == expected
+
+
+class TestLexerComments:
+    def test_single_line_comment(self, lex):
+        tokens = lex("# this is a comment\nx = 1")
+        assert tokens[0].type == TokenType.NEWLINE
+        assert tokens[1].type == TokenType.IDENTIFIER
+        assert tokens[1].value == "x"
+
+    def test_comment_at_end(self, lex):
+        tokens = lex("x = 1 # comment")
+        assert tokens[0].type == TokenType.IDENTIFIER
+        assert tokens[1].type == TokenType.ASSIGN
+        assert tokens[2].type == TokenType.INTEGER
+
+
+class TestLexerLineInfo:
+    def test_line_tracking(self, lex):
+        tokens = lex("x\ny\nz")
+        assert tokens[0].line == 1
+        assert tokens[2].line == 2
+        assert tokens[4].line == 3
+
+    def test_column_tracking(self, lex):
+        tokens = lex("abc def")
+        assert tokens[0].column == 1
+        assert tokens[1].column == 5
+
+
+class TestLexerComplexCases:
+    def test_function_declaration(self, lex):
+        tokens = lex("func greet(name) { }")
+        types = [t.type for t in tokens if t.type != TokenType.EOF]
+        assert TokenType.FUNC in types
+        assert TokenType.IDENTIFIER in types
+        assert TokenType.LPAREN in types
+        assert TokenType.RPAREN in types
+        assert TokenType.LBRACE in types
+        assert TokenType.RBRACE in types
+
+    def test_assignment_expression(self, lex):
+        tokens = lex("x = 42 + y")
+        assert tokens[0].type == TokenType.IDENTIFIER
+        assert tokens[1].type == TokenType.ASSIGN
+        assert tokens[2].type == TokenType.INTEGER
+        assert tokens[3].type == TokenType.PLUS
+        assert tokens[4].type == TokenType.IDENTIFIER
+
+    def test_lambda_arrow(self, lex):
+        tokens = lex("x => x * 2")
+        assert tokens[0].type == TokenType.IDENTIFIER
+        assert tokens[1].type == TokenType.ARROW
+        assert tokens[2].type == TokenType.IDENTIFIER
+        assert tokens[3].type == TokenType.STAR
+        assert tokens[4].type == TokenType.INTEGER