From 32cecbfe4adc5bcb60435c3baa5dcd84df6320e7 Mon Sep 17 00:00:00 2001 From: Aniket Date: Sat, 26 Aug 2023 16:35:50 -0400 Subject: [PATCH] freeing tokenizer in test.c --- test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test.c b/test.c index 23fca8c..4203efd 100644 --- a/test.c +++ b/test.c @@ -73,6 +73,9 @@ void test_prompt_encodings() { char* prompt4 = "Translate English to French:\n\n sea otter => loutre de mer\n peppermint => menthe poivrée\n plush girafe => girafe peluche\n cheese =>"; int expected_tokens4[] = {1, 4103, 9632, 4223, 304, 5176, 29901, 13, 13, 4706, 7205, 4932, 357, 1149, 301, 449, 276, 316, 2778, 13, 4706, 1236, 407, 837, 524, 1149, 6042, 354, 772, 440, 29878, 1318, 13, 4706, 715, 1878, 330, 3055, 1725, 1149, 330, 3055, 1725, 4639, 28754, 13, 4706, 923, 968, 1149}; test_prompt_encoding(&tokenizer, prompt4, expected_tokens4, sizeof(expected_tokens4) / sizeof(int)); + + // memory and file handles cleanup + free_tokenizer(&tokenizer); } int main(int argc, char *argv[]) {