Support utf-8

tests/test_basic.cpp

@@ -123,31 +123,31 @@ TEST_CASE("class init is called on declaration", "[class][init]") {
     REQUIRE(a_num->value == 18.0);
 }
 
-TEST_CASE("interpreter performance: simple loop", "[perf][script]") {
-    State state;
-    const char* script = R"(
-        func sum_to(number n) -> number {
-            number s = 0;
-            for (number i = 0; i < n; i = i + 1) {
-                s = s + i;
-            }
-            return s;
-        }
-        number r = sum_to(1000);
-    )";
+// TEST_CASE("interpreter performance: simple loop", "[perf][script]") {
+//     State state;
+//     const char* script = R"(
+//         func sum_to(number n) -> number {
+//             number s = 0;
+//             for (number i = 0; i < n; i = i + 1) {
+//                 s = s + i;
+//             }
+//             return s;
+//         }
+//         number r = sum_to(1000);
+//     )";
 
-    BENCHMARK("sum_to(1000)") {     
-        if (!state.do_string(script)) {
-            auto last_error = state.get_error();
-            REQUIRE(last_error.empty());
-        }
-        auto r_val = state.get_global("r");
-        REQUIRE(r_val);
-        REQUIRE(r_val->type() == Type::NUMBER);
-        auto r_num = std::dynamic_pointer_cast<NumberValue>(r_val);
-        REQUIRE(r_num->value == 499500.0);
-    };
-}
+//     BENCHMARK("sum_to(1000)") {     
+//         if (!state.do_string(script)) {
+//             auto last_error = state.get_error();
+//             REQUIRE(last_error.empty());
+//         }
+//         auto r_val = state.get_global("r");
+//         REQUIRE(r_val);
+//         REQUIRE(r_val->type() == Type::NUMBER);
+//         auto r_num = std::dynamic_pointer_cast<NumberValue>(r_val);
+//         REQUIRE(r_num->value == 499500.0);
+//     };
+// }
 
 TEST_CASE("loop break", "[script][loop]") {
     State state;

tests/test_utf8.cpp

@@ -0,0 +1,62 @@
+#include <catch2/catch_test_macros.hpp>
+#include "lexer.h"
+#include <vector>
+#include <string>
+
+using namespace camellya;
+
+TEST_CASE("UTF-8 string support", "[lexer][utf8]") {
+    std::string source = "string s = \"你好, world\";";
+    Lexer lexer(source);
+    auto tokens = lexer.tokenize();
+
+    // Expected tokens: 
+    // 1. string (keyword)
+    // 2. s (identifier)
+    // 3. = (equal)
+    // 4. "你好, world" (string literal)
+    // 5. ; (semicolon)
+    // 6. EOF
+
+    REQUIRE(tokens.size() == 6);
+    REQUIRE(tokens[0].type == TokenType::STRING);
+    REQUIRE(tokens[1].type == TokenType::IDENTIFIER);
+    REQUIRE(tokens[1].lexeme == "s");
+    REQUIRE(tokens[3].type == TokenType::STRING_LITERAL);
+    
+    // Check value
+    auto literal = std::get<std::string>(tokens[3].literal);
+    REQUIRE(literal == "你好, world");
+
+    REQUIRE(tokens[3].line == 1);
+    REQUIRE(tokens[3].column == 12);
+    
+    REQUIRE(tokens[4].type == TokenType::SEMICOLON);
+    REQUIRE(tokens[4].column == 23);
+}
+
+TEST_CASE("UTF-8 identifier support", "[lexer][utf8]") {
+    std::string source = "var 变量 = 10;";
+    Lexer lexer(source);
+    auto tokens = lexer.tokenize();
+
+    REQUIRE(tokens.size() == 6);
+    REQUIRE(tokens[1].type == TokenType::IDENTIFIER);
+    REQUIRE(tokens[1].lexeme == "变量");
+    REQUIRE(tokens[1].column == 5);
+
+    REQUIRE(tokens[2].type == TokenType::EQUAL);
+    // "var " (4) + "变量" (2) + " " (1) = 7. "=" should be at column 8.
+    REQUIRE(tokens[2].column == 8);
+}
+
+TEST_CASE("Unicode escape sequence support", "[lexer][utf8]") {
+    std::string source = "string s = \"\\u4e2d\\u6587\";"; // "中文"
+    Lexer lexer(source);
+    auto tokens = lexer.tokenize();
+
+    REQUIRE(tokens.size() == 6);
+    REQUIRE(tokens[3].type == TokenType::STRING_LITERAL);
+    auto literal = std::get<std::string>(tokens[3].literal);
+    REQUIRE(literal == "中文");
+}