#include "tkc/tokenizer.h" #include "gtest/gtest.h" #include using std::string; TEST(Tokenizer, basic) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init(&tokenizer, "w10 h10 10 3 123.1", 0xffffff, " "); ASSERT_EQ(t, &tokenizer); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("w10"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("h10"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(10, tokenizer_next_int(t, 0)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(3, tokenizer_next_int(t, 0)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(123.1f, tokenizer_next_float(t, 0)); ASSERT_EQ(tokenizer_has_more(t), FALSE); ASSERT_EQ(1.0f, tokenizer_next_float(t, 1)); tokenizer_deinit(t); } TEST(Tokenizer, separators) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init(&tokenizer, "w10, h10, 10 , 3, 123.1", 0xffffff, " ,"); ASSERT_EQ(t, &tokenizer); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("w10"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("h10"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(10, tokenizer_next_int(t, 0)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(3, tokenizer_next_int(t, 0)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(123.1f, tokenizer_next_float(t, 0)); tokenizer_deinit(t); } TEST(Tokenizer, single_char_token) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{a, b=2}", 100, "{ }", "=,"); ASSERT_EQ(t, &tokenizer); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("a"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string(","), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("b"), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("="), tokenizer_next(t)); ASSERT_EQ(tokenizer_has_more(t), TRUE); ASSERT_EQ(string("2"), tokenizer_next(t)); tokenizer_deinit(t); } TEST(Tokenizer, until1) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{a==1}", 100, "{ }", "=,"); ASSERT_EQ(t, &tokenizer); ASSERT_EQ(string(tokenizer_next_until(t, ",}")), string("a==1")); tokenizer_deinit(t); } TEST(Tokenizer, until2) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{a==1, a}", 100, "{ }", "=,"); ASSERT_EQ(t, &tokenizer); ASSERT_EQ(string(tokenizer_next_until(t, ",}")), string("a==1")); tokenizer_deinit(t); } TEST(Tokenizer, expr1) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{\"a,a\"}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("\"a,a\"")); tokenizer_deinit(t); } TEST(Tokenizer, expr2) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{\"{\\\"}\"}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("\"{\\\"}\"")); tokenizer_deinit(t); } TEST(Tokenizer, expr3) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{(1+2)+\"aaa\"}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("(1+2)+\"aaa\"")); tokenizer_deinit(t); } TEST(Tokenizer, expr4) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{format(\"%d\", 100)}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("format(\"%d\", 100)")); tokenizer_deinit(t); } TEST(Tokenizer, expr5) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{a, name=aaa}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("a")); tokenizer_deinit(t); } TEST(Tokenizer, expr6) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{(a+b), name=aaa}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("(a+b)")); tokenizer_deinit(t); } TEST(Tokenizer, expr7) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex( &tokenizer, "{fformat(tr(\"current temperature %f\"), $value)+\"c\"}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("fformat(tr(\"current temperature %f\"), $value)+\"c\"")); tokenizer_deinit(t); } TEST(Tokenizer, expr8) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex(&tokenizer, "{a(b(1,2),c(2,3),d(5,6), e(\"()\"))),}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("a(b(1,2),c(2,3),d(5,6), e(\"()\")))")); tokenizer_deinit(t); } TEST(Tokenizer, expr9) { tokenizer_t tokenizer; tokenizer_t* t = tokenizer_init_ex( &tokenizer, "{fformat(\"%2.2lf F\", round($value * 1.8 + 32))}", 100, "{}", "=,"); ASSERT_EQ(string(tokenizer_next_expr_until(t, ",}")), string("fformat(\"%2.2lf F\", round($value * 1.8 + 32))")); tokenizer_deinit(t); } TEST(Tokenizer, str) { tokenizer_t tokenizer; const char* input = "a='a=b && c=d',b=123,c=\"e=123\",d='123,234'"; tokenizer_t* t = tokenizer_init(&tokenizer, input, strlen(input), ",="); ASSERT_STREQ(tokenizer_next(t), "a"); ASSERT_STREQ(tokenizer_next_str(t), "a=b && c=d"); ASSERT_STREQ(tokenizer_next(t), "b"); ASSERT_STREQ(tokenizer_next_str(t), "123"); ASSERT_STREQ(tokenizer_next(t), "c"); ASSERT_STREQ(tokenizer_next_str(t), "e=123"); ASSERT_STREQ(tokenizer_next(t), "d"); ASSERT_STREQ(tokenizer_next(t), "'123"); tokenizer_deinit(t); }