From bab1812cc9e2959920309554bf6a26650dd4032b Mon Sep 17 00:00:00 2001
From: Laria Carolin Chabowski <laria@laria.me>
Date: Thu, 6 Jan 2022 22:50:44 +0100
Subject: [PATCH] Add tokenizer tests to check binary string handling

---
 src/tokenizer_test.c | 37 +++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/src/tokenizer_test.c b/src/tokenizer_test.c
index f8a3e17..e6510f5 100644
--- a/src/tokenizer_test.c
+++ b/src/tokenizer_test.c
@@ -11,9 +11,9 @@ struct tokenizer_test {
 };
 
 static struct tokenizer_test *
-new_tokenizer_test(testctx t, const char *text)
+new_tokenizer_test_sv(testctx t, struct apfl_string_view text)
 {
-    void *ctx = apfl_string_source_reader_new(apfl_string_view_from(text));
+    void *ctx = apfl_string_source_reader_new(text);
     if (ctx == NULL) {
         test_fatalf(t, "Failed to initialize the source reader");
     }
@@ -34,6 +34,13 @@ new_tokenizer_test(testctx t, const char *text)
     return tt;
 }
 
+static struct tokenizer_test *
+new_tokenizer_test(testctx t, const char *text)
+{
+    return new_tokenizer_test_sv(t, apfl_string_view_from(text));
+}
+
+
 static void
 destroy_tokenizer_test(struct tokenizer_test *tt)
 {
@@ -116,6 +123,18 @@ expect_text_token(struct tokenizer_test *tt, int line, int col, enum apfl_token_
     }
 }
 
+static void
+expect_text_token_sv(struct tokenizer_test *tt, int line, int col, enum apfl_token_type type, struct apfl_string_view text)
+{
+    struct apfl_token tok;
+    if (expect_token(tt, line, col, type, &tok)) {
+        if (!apfl_string_eq(text, tok.text)) {
+            test_failf(tt->t, "Token has wrong content. have=\"" APFL_STR_FMT "\", want=\"%s\"", APFL_STR_FMT_ARGS(tok.text), text);
+        }
+        apfl_token_deinit(&tok);
+    }
+}
+
 static void
 expect_number_token(struct tokenizer_test *tt, int line, int col, enum apfl_token_type type, apfl_number num)
 {
@@ -251,6 +270,19 @@ TEST(all_tokens, t) {
     destroy_tokenizer_test(tt);
 }
 
+TEST(strings_with_binary_data, t) {
+    struct tokenizer_test *tt = new_tokenizer_test_sv(t, (struct apfl_string_view) {
+        .bytes = (char []){'"', '\x00', '\xFF', '\\', 'x', '0', '0', '\\', 'x', '2', 'a', '"'},
+        .len = 12,
+    });
+    expect_text_token_sv(tt, 1, 1, APFL_TOK_STRING, (struct apfl_string_view) {
+        .bytes = (char []){'\x00', '\xFF', '\x00', '\x2A'},
+        .len = 4
+    });
+    expect_eof(tt);
+    destroy_tokenizer_test(tt);
+}
+
 TESTS_BEGIN
     ADDTEST(empty),
     ADDTEST(simple_variable),
@@ -258,4 +290,5 @@ TESTS_BEGIN
     ADDTEST(names),
     ADDTEST(assignment),
     ADDTEST(all_tokens),
+    ADDTEST(strings_with_binary_data),
 TESTS_END