From 9b66a6b8b93cbc7706800ee7ac3078ed6f56b589 Mon Sep 17 00:00:00 2001 From: Oliver Steele Date: Fri, 27 Feb 2026 17:09:31 +0800 Subject: [PATCH] feat: add escape character support in double-quoted string literals Closes #45 --- expressions/scanner.go | 505 ++++++++++++++++++------------------ expressions/scanner.rl | 45 +++- expressions/scanner_test.go | 53 ++++ 3 files changed, 343 insertions(+), 260 deletions(-) diff --git a/expressions/scanner.go b/expressions/scanner.go index 0c2aff4..c9a5451 100644 --- a/expressions/scanner.go +++ b/expressions/scanner.go @@ -5,10 +5,11 @@ package expressions import ( "strconv" + "strings" "unicode" ) -//line scanner.go:12 +//line scanner.go:13 var _expression_actions []byte = []byte{ 0, 1, 0, 1, 1, 1, 2, 1, 12, 1, 13, 1, 14, 1, 15, 1, 16, @@ -23,242 +24,241 @@ var _expression_actions []byte = []byte{ } var _expression_key_offsets []int16 = []int16{ - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 14, 16, 18, - 20, 22, 24, 26, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 70, 73, - 74, 75, 77, 78, 80, 83, 85, 97, - 112, 113, 114, 115, 131, 132, 149, 166, - 183, 200, 217, 234, 251, 268, 285, 302, - 319, 336, 353, 370, 387, 404, 421, 438, - 455, 456, 458, 460, + 0, 2, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 15, 17, + 19, 21, 23, 25, 27, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 71, + 74, 75, 77, 79, 80, 82, 85, 87, + 99, 114, 115, 116, 117, 133, 134, 151, + 168, 185, 202, 219, 236, 253, 270, 287, + 304, 321, 338, 355, 372, 389, 406, 423, + 440, 457, 458, 460, 462, } var _expression_trans_keys []byte = []byte{ - 34, 115, 115, 105, 103, 110, 32, 111, - 111, 112, 32, 39, 48, 57, 128, 191, - 128, 191, 128, 191, 128, 191, 128, 191, - 128, 191, 99, 119, 121, 99, 108, 101, - 32, 104, 101, 110, 32, 32, 33, 34, - 37, 39, 45, 46, 60, 61, 62, 95, - 97, 99, 102, 105, 110, 111, 116, 123, - 9, 13, 48, 57, 65, 90, 98, 122, - 194, 223, 224, 239, 240, 247, 32, 9, - 13, 61, 34, 97, 108, 39, 48, 57, - 46, 48, 57, 48, 57, 46, 95, 65, - 90, 97, 122, 194, 223, 224, 239, 240, - 247, 45, 63, 95, 48, 57, 65, 90, - 97, 122, 194, 223, 224, 239, 240, 247, - 61, 61, 61, 45, 58, 63, 95, 48, - 57, 65, 90, 97, 122, 194, 223, 224, - 239, 240, 247, 58, 45, 58, 63, 95, - 110, 48, 57, 65, 90, 97, 122, 194, - 223, 224, 239, 240, 247, 45, 58, 63, - 95, 100, 48, 57, 65, 90, 97, 122, - 194, 223, 224, 239, 240, 247, 45, 58, - 63, 95, 111, 48, 57, 65, 90, 97, + 34, 92, 115, 115, 105, 103, 110, 32, + 111, 111, 112, 32, 39, 48, 57, 128, + 191, 128, 191, 128, 191, 128, 191, 128, + 191, 128, 191, 99, 119, 121, 99, 108, + 101, 32, 104, 101, 110, 32, 32, 33, + 34, 37, 39, 45, 46, 60, 61, 62, + 95, 97, 99, 102, 105, 110, 111, 116, + 123, 9, 13, 48, 57, 65, 90, 98, + 122, 194, 223, 224, 239, 240, 247, 32, + 9, 13, 61, 34, 92, 97, 108, 39, + 48, 57, 46, 48, 57, 48, 57, 46, + 95, 65, 90, 97, 122, 194, 223, 224, + 239, 240, 247, 45, 63, 95, 48, 57, + 65, 90, 97, 122, 194, 223, 224, 239, + 240, 247, 61, 61, 61, 45, 58, 63, + 95, 48, 57, 65, 90, 97, 122, 194, + 223, 224, 239, 240, 247, 58, 45, 58, + 63, 95, 110, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, - 58, 63, 95, 110, 48, 57, 65, 90, + 58, 63, 95, 100, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, - 45, 58, 63, 95, 116, 48, 57, 65, + 45, 58, 63, 95, 111, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, - 247, 45, 58, 63, 95, 97, 48, 57, - 65, 90, 98, 122, 194, 223, 224, 239, - 240, 247, 45, 58, 63, 95, 105, 48, + 247, 45, 58, 63, 95, 110, 48, 57, + 65, 90, 97, 122, 194, 223, 224, 239, + 240, 247, 45, 58, 63, 95, 116, 48, 57, 65, 90, 97, 122, 194, 223, 224, - 239, 240, 247, 45, 58, 63, 95, 110, - 48, 57, 65, 90, 97, 122, 194, 223, + 239, 240, 247, 45, 58, 63, 95, 97, + 48, 57, 65, 90, 98, 122, 194, 223, 224, 239, 240, 247, 45, 58, 63, 95, - 115, 48, 57, 65, 90, 97, 122, 194, + 105, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, 58, 63, - 95, 97, 48, 57, 65, 90, 98, 122, + 95, 110, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, 58, - 63, 95, 108, 48, 57, 65, 90, 97, + 63, 95, 115, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, - 58, 63, 95, 115, 48, 57, 65, 90, - 97, 122, 194, 223, 224, 239, 240, 247, - 45, 58, 63, 95, 101, 48, 57, 65, + 58, 63, 95, 97, 48, 57, 65, 90, + 98, 122, 194, 223, 224, 239, 240, 247, + 45, 58, 63, 95, 108, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, - 247, 45, 58, 63, 95, 110, 48, 57, + 247, 45, 58, 63, 95, 115, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, - 240, 247, 45, 58, 63, 95, 105, 48, + 240, 247, 45, 58, 63, 95, 101, 48, 57, 65, 90, 97, 122, 194, 223, 224, - 239, 240, 247, 45, 58, 63, 95, 108, + 239, 240, 247, 45, 58, 63, 95, 110, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, 58, 63, 95, - 114, 48, 57, 65, 90, 97, 122, 194, + 105, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, 58, 63, - 95, 114, 48, 57, 65, 90, 97, 122, + 95, 108, 48, 57, 65, 90, 97, 122, 194, 223, 224, 239, 240, 247, 45, 58, - 63, 95, 117, 48, 57, 65, 90, 97, - 122, 194, 223, 224, 239, 240, 247, 37, - 128, 191, 128, 191, 128, 191, + 63, 95, 114, 48, 57, 65, 90, 97, + 122, 194, 223, 224, 239, 240, 247, 45, + 58, 63, 95, 114, 48, 57, 65, 90, + 97, 122, 194, 223, 224, 239, 240, 247, + 45, 58, 63, 95, 117, 48, 57, 65, + 90, 97, 122, 194, 223, 224, 239, 240, + 247, 37, 128, 191, 128, 191, 128, 191, } var _expression_single_lengths []byte = []byte{ - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 0, 0, 0, 0, - 0, 0, 0, 2, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 19, 1, 1, - 1, 2, 1, 0, 1, 0, 2, 3, - 1, 1, 1, 4, 1, 5, 5, 5, + 2, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 19, 1, + 1, 2, 2, 1, 0, 1, 0, 2, + 3, 1, 1, 1, 4, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 1, 0, 0, 0, + 5, 1, 0, 0, 0, } var _expression_range_lengths []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 1, 1, 1, - 1, 1, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 7, 1, 0, - 0, 0, 0, 1, 1, 1, 5, 6, - 0, 0, 0, 6, 0, 6, 6, 6, + 0, 0, 0, 0, 0, 1, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 7, 1, + 0, 0, 0, 0, 1, 1, 1, 5, + 6, 0, 0, 0, 6, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - 0, 1, 1, 1, + 6, 0, 1, 1, 1, } var _expression_index_offsets []int16 = []int16{ - 0, 2, 4, 6, 8, 10, 12, 14, + 0, 3, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, - 32, 34, 36, 38, 41, 43, 45, 47, - 49, 51, 53, 55, 57, 59, 86, 89, - 91, 93, 96, 98, 100, 103, 105, 113, - 123, 125, 127, 129, 140, 142, 154, 166, - 178, 190, 202, 214, 226, 238, 250, 262, - 274, 286, 298, 310, 322, 334, 346, 358, - 370, 372, 374, 376, + 32, 34, 36, 38, 40, 43, 45, 47, + 49, 51, 53, 55, 57, 59, 61, 88, + 91, 93, 96, 99, 101, 103, 106, 108, + 116, 126, 128, 130, 132, 143, 145, 157, + 169, 181, 193, 205, 217, 229, 241, 253, + 265, 277, 289, 301, 313, 325, 337, 349, + 361, 373, 375, 377, 379, } var _expression_indicies []byte = []byte{ - 2, 1, 3, 0, 4, 0, 5, 0, + 2, 3, 1, 1, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, 9, 0, - 10, 0, 11, 0, 12, 0, 2, 13, - 15, 14, 17, 16, 18, 16, 19, 16, - 20, 16, 21, 16, 22, 16, 23, 24, - 0, 25, 0, 26, 0, 27, 0, 28, + 10, 0, 11, 0, 12, 0, 13, 0, + 2, 14, 16, 15, 18, 17, 19, 17, + 20, 17, 21, 17, 22, 17, 23, 17, + 24, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0, 32, - 0, 33, 0, 35, 36, 37, 38, 39, - 40, 41, 43, 44, 45, 20, 46, 47, - 48, 49, 50, 51, 52, 53, 35, 42, - 20, 20, 54, 55, 56, 34, 35, 35, - 57, 59, 58, 2, 1, 60, 61, 58, - 2, 13, 42, 58, 63, 42, 62, 15, - 64, 65, 17, 17, 17, 18, 19, 66, - 58, 17, 68, 17, 17, 17, 17, 18, - 19, 66, 67, 69, 58, 70, 58, 71, - 58, 20, 72, 73, 20, 20, 20, 20, - 21, 22, 74, 16, 72, 75, 20, 72, - 73, 20, 76, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 77, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 78, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 79, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 80, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 81, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 82, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 83, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 84, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 85, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 86, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 87, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 88, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 89, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 90, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 91, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 92, 20, 20, 20, 21, 22, - 74, 75, 20, 72, 73, 20, 93, 20, - 20, 20, 21, 22, 74, 75, 20, 72, - 73, 20, 87, 20, 20, 20, 21, 22, - 74, 75, 94, 58, 20, 58, 21, 58, - 22, 58, + 0, 33, 0, 34, 0, 36, 37, 38, + 39, 40, 41, 42, 44, 45, 46, 21, + 47, 48, 49, 50, 51, 52, 53, 54, + 36, 43, 21, 21, 55, 56, 57, 35, + 36, 36, 58, 60, 59, 2, 3, 1, + 61, 62, 59, 2, 14, 43, 59, 64, + 43, 63, 16, 65, 66, 18, 18, 18, + 19, 20, 67, 59, 18, 69, 18, 18, + 18, 18, 19, 20, 67, 68, 70, 59, + 71, 59, 72, 59, 21, 73, 74, 21, + 21, 21, 21, 22, 23, 75, 17, 73, + 76, 21, 73, 74, 21, 77, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 78, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 79, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 80, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 81, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 82, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 83, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 84, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 85, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 86, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 87, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 88, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 89, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 90, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 91, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 92, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 93, 21, 21, + 21, 22, 23, 75, 76, 21, 73, 74, + 21, 94, 21, 21, 21, 22, 23, 75, + 76, 21, 73, 74, 21, 88, 21, 21, + 21, 22, 23, 75, 76, 95, 59, 21, + 59, 22, 59, 23, 59, } var _expression_trans_targs []byte = []byte{ - 29, 0, 29, 2, 3, 4, 5, 6, - 29, 8, 9, 10, 29, 11, 29, 37, - 29, 39, 13, 14, 43, 16, 17, 20, - 25, 21, 22, 23, 24, 29, 26, 27, - 28, 29, 29, 30, 31, 32, 33, 34, - 35, 38, 36, 40, 41, 42, 45, 47, - 54, 58, 59, 61, 62, 64, 65, 66, - 67, 29, 29, 29, 1, 7, 29, 12, - 29, 29, 15, 29, 29, 29, 29, 29, - 29, 44, 18, 29, 46, 43, 48, 49, - 50, 51, 52, 53, 43, 55, 56, 57, - 43, 43, 60, 43, 43, 63, 19, + 30, 0, 30, 1, 3, 4, 5, 6, + 7, 30, 9, 10, 11, 30, 12, 30, + 38, 30, 40, 14, 15, 44, 17, 18, + 21, 26, 22, 23, 24, 25, 30, 27, + 28, 29, 30, 30, 31, 32, 33, 34, + 35, 36, 39, 37, 41, 42, 43, 46, + 48, 55, 59, 60, 62, 63, 65, 66, + 67, 68, 30, 30, 30, 2, 8, 30, + 13, 30, 30, 16, 30, 30, 30, 30, + 30, 30, 45, 19, 30, 47, 44, 49, + 50, 51, 52, 53, 54, 44, 56, 57, + 58, 44, 44, 61, 44, 44, 64, 20, } var _expression_trans_actions []byte = []byte{ 47, 0, 15, 0, 0, 0, 0, 0, - 7, 0, 0, 0, 11, 0, 45, 0, - 49, 72, 0, 0, 69, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 0, 0, - 0, 13, 31, 0, 0, 5, 5, 5, - 0, 75, 5, 0, 0, 0, 69, 69, - 69, 69, 69, 69, 69, 5, 0, 75, - 75, 41, 43, 19, 0, 0, 33, 0, - 35, 25, 0, 39, 29, 23, 17, 21, - 27, 0, 0, 37, 69, 57, 69, 69, - 69, 69, 69, 69, 63, 69, 69, 69, - 51, 66, 69, 54, 60, 69, 0, + 0, 7, 0, 0, 0, 11, 0, 45, + 0, 49, 72, 0, 0, 69, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 0, + 0, 0, 13, 31, 0, 0, 5, 5, + 5, 0, 75, 5, 0, 0, 0, 69, + 69, 69, 69, 69, 69, 69, 5, 0, + 75, 75, 41, 43, 19, 0, 0, 33, + 0, 35, 25, 0, 39, 29, 23, 17, + 21, 27, 0, 0, 37, 69, 57, 69, + 69, 69, 69, 69, 69, 63, 69, 69, + 69, 51, 66, 69, 54, 60, 69, 0, } var _expression_to_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, + 0, 0, 0, 0, 0, } var _expression_from_state_actions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, + 0, 0, 0, 0, 0, } var _expression_eof_trans []int16 = []int16{ 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 15, 17, 17, 17, - 17, 17, 17, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 58, 59, - 59, 59, 59, 59, 63, 65, 59, 68, - 59, 59, 59, 17, 76, 76, 76, 76, - 76, 76, 76, 76, 76, 76, 76, 76, - 76, 76, 76, 76, 76, 76, 76, 76, - 59, 59, 59, 59, + 1, 1, 1, 1, 1, 16, 18, 18, + 18, 18, 18, 18, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 0, 59, + 60, 60, 60, 60, 60, 64, 66, 60, + 69, 60, 60, 60, 18, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, + 77, 60, 60, 60, 60, } -const expression_start int = 29 -const expression_first_final int = 29 +const expression_start int = 30 +const expression_first_final int = 30 const expression_error int = -1 -const expression_en_main int = 29 +const expression_en_main int = 30 -//line scanner.rl:26 +//line scanner.rl:27 type lexer struct { parseValue - data []byte p, pe, cs int ts, te, act int @@ -274,7 +274,7 @@ func newLexer(data []byte) *lexer { pe: len(data), } -//line scanner.go:279 +//line scanner.go:281 { lex.cs = expression_start lex.ts = 0 @@ -282,7 +282,7 @@ func newLexer(data []byte) *lexer { lex.act = 0 } -//line scanner.rl:45 +//line scanner.rl:46 return lex } @@ -290,26 +290,19 @@ func (lex *lexer) Lex(out *yySymType) int { eof := lex.pe tok := 0 -//line scanner.go:296 +//line scanner.go:298 { - var ( - _klen int - _trans int - _acts int - _nacts uint - _keys int - ) - + var _klen int + var _trans int + var _acts int + var _nacts uint + var _keys int if (lex.p) == (lex.pe) { goto _test_eof } - _resume: - _acts = int(_expression_from_state_actions[lex.cs]) - _nacts = uint(_expression_actions[_acts]) - _acts++ for ; _nacts > 0; _nacts-- { _acts++ @@ -318,7 +311,7 @@ func (lex *lexer) Lex(out *yySymType) int { //line NONE:1 lex.ts = (lex.p) -//line scanner.go:316 +//line scanner.go:318 } } @@ -328,9 +321,7 @@ func (lex *lexer) Lex(out *yySymType) int { _klen = int(_expression_single_lengths[lex.cs]) if _klen > 0 { _lower := int(_keys) - var _mid int - _upper := int(_keys + _klen - 1) for { if _upper < _lower { @@ -348,7 +339,6 @@ func (lex *lexer) Lex(out *yySymType) int { goto _match } } - _keys += _klen _trans += _klen } @@ -356,9 +346,7 @@ func (lex *lexer) Lex(out *yySymType) int { _klen = int(_expression_range_lengths[lex.cs]) if _klen > 0 { _lower := int(_keys) - var _mid int - _upper := int(_keys + (_klen << 1) - 2) for { if _upper < _lower { @@ -376,15 +364,12 @@ func (lex *lexer) Lex(out *yySymType) int { goto _match } } - _trans += _klen } _match: _trans = int(_expression_indicies[_trans]) - _eof_trans: - lex.cs = int(_expression_trans_targs[_trans]) if _expression_trans_actions[_trans] == 0 { @@ -393,7 +378,6 @@ func (lex *lexer) Lex(out *yySymType) int { _acts = int(_expression_trans_actions[_trans]) _nacts = uint(_expression_actions[_acts]) - _acts++ for ; _nacts > 0; _nacts-- { _acts++ @@ -403,189 +387,177 @@ func (lex *lexer) Lex(out *yySymType) int { lex.te = (lex.p) + 1 case 3: -//line scanner.rl:53 +//line scanner.rl:54 lex.act = 8 case 4: -//line scanner.rl:116 +//line scanner.rl:122 lex.act = 9 case 5: -//line scanner.rl:123 +//line scanner.rl:129 lex.act = 14 case 6: -//line scanner.rl:124 +//line scanner.rl:130 lex.act = 15 case 7: -//line scanner.rl:125 +//line scanner.rl:131 lex.act = 16 case 8: -//line scanner.rl:128 +//line scanner.rl:134 lex.act = 17 case 9: -//line scanner.rl:58 +//line scanner.rl:59 lex.act = 20 case 10: -//line scanner.rl:133 +//line scanner.rl:139 lex.act = 21 case 11: -//line scanner.rl:136 +//line scanner.rl:142 lex.act = 23 case 12: -//line scanner.rl:104 +//line scanner.rl:110 lex.te = (lex.p) + 1 { tok = ASSIGN (lex.p)++ - goto _out } case 13: -//line scanner.rl:105 +//line scanner.rl:111 lex.te = (lex.p) + 1 { tok = CYCLE (lex.p)++ - goto _out } case 14: -//line scanner.rl:106 +//line scanner.rl:112 lex.te = (lex.p) + 1 { tok = LOOP (lex.p)++ - goto _out } case 15: -//line scanner.rl:107 +//line scanner.rl:113 lex.te = (lex.p) + 1 { tok = WHEN (lex.p)++ - goto _out } case 16: -//line scanner.rl:87 +//line scanner.rl:88 lex.te = (lex.p) + 1 { tok = LITERAL - // TODO unescape \x - out.val = string(lex.data[lex.ts+1 : lex.te-1]) + raw := string(lex.data[lex.ts+1 : lex.te-1]) + if lex.data[lex.ts] == '"' { + out.val = unescapeString(raw) + } else { + out.val = raw + } (lex.p)++ - goto _out + } case 17: -//line scanner.rl:119 +//line scanner.rl:125 lex.te = (lex.p) + 1 { tok = EQ (lex.p)++ - goto _out } case 18: -//line scanner.rl:120 +//line scanner.rl:126 lex.te = (lex.p) + 1 { tok = NEQ (lex.p)++ - goto _out } case 19: -//line scanner.rl:121 +//line scanner.rl:127 lex.te = (lex.p) + 1 { tok = GE (lex.p)++ - goto _out } case 20: -//line scanner.rl:122 +//line scanner.rl:128 lex.te = (lex.p) + 1 { tok = LE (lex.p)++ - goto _out } case 21: -//line scanner.rl:129 +//line scanner.rl:135 lex.te = (lex.p) + 1 { tok = DOTDOT (lex.p)++ - goto _out } case 22: -//line scanner.rl:131 +//line scanner.rl:137 lex.te = (lex.p) + 1 { tok = KEYWORD out.name = string(lex.data[lex.ts : lex.te-1]) (lex.p)++ - goto _out } case 23: -//line scanner.rl:133 +//line scanner.rl:139 lex.te = (lex.p) + 1 { tok = PROPERTY out.name = string(lex.data[lex.ts+1 : lex.te]) (lex.p)++ - goto _out } case 24: -//line scanner.rl:136 +//line scanner.rl:142 lex.te = (lex.p) + 1 { tok = int(lex.data[lex.ts]) (lex.p)++ - goto _out } case 25: -//line scanner.rl:69 +//line scanner.rl:70 lex.te = (lex.p) (lex.p)-- { tok = LITERAL - n, err := strconv.ParseInt(lex.token(), 10, 64) if err != nil { panic(err) } - out.val = int(n) (lex.p)++ - goto _out + } case 26: -//line scanner.rl:78 +//line scanner.rl:79 lex.te = (lex.p) (lex.p)-- { tok = LITERAL - n, err := strconv.ParseFloat(lex.token(), 64) if err != nil { panic(err) } - out.val = n (lex.p)++ - goto _out + } case 27: -//line scanner.rl:58 +//line scanner.rl:59 lex.te = (lex.p) (lex.p)-- { @@ -598,58 +570,53 @@ func (lex *lexer) Lex(out *yySymType) int { out.name = t (lex.p)++ - goto _out + } case 28: -//line scanner.rl:133 +//line scanner.rl:139 lex.te = (lex.p) (lex.p)-- { tok = PROPERTY out.name = string(lex.data[lex.ts+1 : lex.te]) (lex.p)++ - goto _out } case 29: -//line scanner.rl:135 +//line scanner.rl:141 lex.te = (lex.p) (lex.p)-- case 30: -//line scanner.rl:136 +//line scanner.rl:142 lex.te = (lex.p) (lex.p)-- { tok = int(lex.data[lex.ts]) (lex.p)++ - goto _out } case 31: -//line scanner.rl:69 +//line scanner.rl:70 (lex.p) = (lex.te) - 1 { tok = LITERAL - n, err := strconv.ParseInt(lex.token(), 10, 64) if err != nil { panic(err) } - out.val = int(n) (lex.p)++ - goto _out + } case 32: -//line scanner.rl:136 +//line scanner.rl:142 (lex.p) = (lex.te) - 1 { tok = int(lex.data[lex.ts]) (lex.p)++ - goto _out } case 33: @@ -662,8 +629,8 @@ func (lex *lexer) Lex(out *yySymType) int { tok = LITERAL out.val = lex.token() == "true" (lex.p)++ - goto _out + } case 9: { @@ -671,7 +638,6 @@ func (lex *lexer) Lex(out *yySymType) int { tok = LITERAL out.val = nil (lex.p)++ - goto _out } case 14: @@ -679,7 +645,6 @@ func (lex *lexer) Lex(out *yySymType) int { (lex.p) = (lex.te) - 1 tok = AND (lex.p)++ - goto _out } case 15: @@ -687,7 +652,6 @@ func (lex *lexer) Lex(out *yySymType) int { (lex.p) = (lex.te) - 1 tok = OR (lex.p)++ - goto _out } case 16: @@ -695,7 +659,6 @@ func (lex *lexer) Lex(out *yySymType) int { (lex.p) = (lex.te) - 1 tok = CONTAINS (lex.p)++ - goto _out } case 17: @@ -703,7 +666,6 @@ func (lex *lexer) Lex(out *yySymType) int { (lex.p) = (lex.te) - 1 tok = IN (lex.p)++ - goto _out } case 20: @@ -719,8 +681,8 @@ func (lex *lexer) Lex(out *yySymType) int { out.name = t (lex.p)++ - goto _out + } case 21: { @@ -739,15 +701,13 @@ func (lex *lexer) Lex(out *yySymType) int { } } -//line scanner.go:621 +//line scanner.go:627 } } _again: _acts = int(_expression_to_state_actions[lex.cs]) - _nacts = uint(_expression_actions[_acts]) - _acts++ for ; _nacts > 0; _nacts-- { _acts++ @@ -756,7 +716,7 @@ func (lex *lexer) Lex(out *yySymType) int { //line NONE:1 lex.ts = 0 -//line scanner.go:635 +//line scanner.go:641 } } @@ -764,11 +724,9 @@ func (lex *lexer) Lex(out *yySymType) int { if (lex.p) != (lex.pe) { goto _resume } - _test_eof: { } - if (lex.p) == eof { if _expression_eof_trans[lex.cs] > 0 { _trans = int(_expression_eof_trans[lex.cs] - 1) @@ -781,7 +739,7 @@ func (lex *lexer) Lex(out *yySymType) int { } } -//line scanner.rl:140 +//line scanner.rl:146 return tok } @@ -790,6 +748,39 @@ func (lex *lexer) Error(e string) { // fmt.Println("scan error:", e) } +// unescapeString processes escape sequences in double-quoted strings. +// Supported: \\, \", \n, \t, \r. Unknown sequences pass through as-is. +func unescapeString(s string) string { + if !strings.ContainsRune(s, '\\') { + return s + } + var b strings.Builder + b.Grow(len(s)) + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+1 < len(s) { + i++ + switch s[i] { + case '\\': + b.WriteByte('\\') + case '"': + b.WriteByte('"') + case 'n': + b.WriteByte('\n') + case 't': + b.WriteByte('\t') + case 'r': + b.WriteByte('\r') + default: + b.WriteByte('\\') + b.WriteByte(s[i]) + } + } else { + b.WriteByte(s[i]) + } + } + return b.String() +} + func isValidUnicodeIdentifier(s string) bool { // Remove optional trailing '?' for validation checkStr := s diff --git a/expressions/scanner.rl b/expressions/scanner.rl index e559a87..4e0abb3 100644 --- a/expressions/scanner.rl +++ b/expressions/scanner.rl @@ -2,6 +2,7 @@ package expressions import ( "strconv" + "strings" "unicode" ) @@ -86,8 +87,12 @@ func (lex *lexer) Lex(out *yySymType) int { } action String { tok = LITERAL - // TODO unescape \x - out.val = string(lex.data[lex.ts+1:lex.te-1]) + raw := string(lex.data[lex.ts+1:lex.te-1]) + if lex.data[lex.ts] == '"' { + out.val = unescapeString(raw) + } else { + out.val = raw + } fbreak; } action Relation { tok = RELATION; out.name = lex.token(); fbreak; } @@ -97,7 +102,8 @@ func (lex *lexer) Lex(out *yySymType) int { int = '-'? digit+ ; float = '-'? digit+ ('.' digit+)? ; - string = '"' (any - '"')* '"' | "'" (any - "'")* "'" ; # TODO escapes + dq_char = (any - '"' - '\\') | ('\\' any); + string = '"' dq_char* '"' | "'" (any - "'")* "'" ; main := |* # statement selectors, should match constants in parser.go @@ -146,6 +152,39 @@ func (lex *lexer) Error(e string) { // fmt.Println("scan error:", e) } +// unescapeString processes escape sequences in double-quoted strings. +// Supported: \\, \", \n, \t, \r. Unknown sequences pass through as-is. +func unescapeString(s string) string { + if !strings.ContainsRune(s, '\\') { + return s + } + var b strings.Builder + b.Grow(len(s)) + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+1 < len(s) { + i++ + switch s[i] { + case '\\': + b.WriteByte('\\') + case '"': + b.WriteByte('"') + case 'n': + b.WriteByte('\n') + case 't': + b.WriteByte('\t') + case 'r': + b.WriteByte('\r') + default: + b.WriteByte('\\') + b.WriteByte(s[i]) + } + } else { + b.WriteByte(s[i]) + } + } + return b.String() +} + func isValidUnicodeIdentifier(s string) bool { // Remove optional trailing '?' for validation checkStr := s diff --git a/expressions/scanner_test.go b/expressions/scanner_test.go index 8dc37dd..bf6bc95 100644 --- a/expressions/scanner_test.go +++ b/expressions/scanner_test.go @@ -96,6 +96,59 @@ func TestLex(t *testing.T) { // require.Len(t, ts, 9) } +func TestLexStringEscapes(t *testing.T) { + // Double-quoted strings support escape sequences + t.Run("backslash", func(t *testing.T) { + ts := scanExpression(`"a\\b"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, `a\b`, ts[0].typ.val) + }) + t.Run("escaped_quote", func(t *testing.T) { + ts := scanExpression(`"say \"hello\""`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, `say "hello"`, ts[0].typ.val) + }) + t.Run("newline", func(t *testing.T) { + ts := scanExpression(`"line1\nline2"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, "line1\nline2", ts[0].typ.val) + }) + t.Run("tab", func(t *testing.T) { + ts := scanExpression(`"col1\tcol2"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, "col1\tcol2", ts[0].typ.val) + }) + t.Run("carriage_return", func(t *testing.T) { + ts := scanExpression(`"a\rb"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, "a\rb", ts[0].typ.val) + }) + t.Run("unknown_escape_passthrough", func(t *testing.T) { + ts := scanExpression(`"a\xb"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, `a\xb`, ts[0].typ.val) + }) + // Single-quoted strings do NOT process escapes + t.Run("single_quote_no_escape", func(t *testing.T) { + ts := scanExpression(`'a\nb'`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, `a\nb`, ts[0].typ.val) + }) + t.Run("no_escapes_plain", func(t *testing.T) { + ts := scanExpression(`"hello"`) + require.Len(t, ts, 1) + require.Equal(t, LITERAL, ts[0].tok) + require.Equal(t, "hello", ts[0].typ.val) + }) +} + func TestLexUnicodeIdentifiers(t *testing.T) { // Test Bengali t.Run("Bengali", func(t *testing.T) {