From 7342b383dcdfa7fcc9d5874b2bb0975793d79c51 Mon Sep 17 00:00:00 2001 From: Jeff Date: Mon, 23 Oct 2023 15:01:00 -0400 Subject: [PATCH] Fix dot notation; Add corpus tests; Clean up --- examples/assets/faithful.csv | 273 ------------------------ examples/fetch.ds | 6 +- src/abstract_tree/function_call.rs | 1 - src/value/mod.rs | 26 +-- src/value/variable_map.rs | 2 +- tree-sitter-dust/corpus/identifiers.txt | 37 ++++ tree-sitter-dust/grammar.js | 2 +- tree-sitter-dust/src/grammar.json | 2 +- tree-sitter-dust/src/parser.c | 93 ++++---- 9 files changed, 106 insertions(+), 336 deletions(-) delete mode 100644 examples/assets/faithful.csv create mode 100644 tree-sitter-dust/corpus/identifiers.txt diff --git a/examples/assets/faithful.csv b/examples/assets/faithful.csv deleted file mode 100644 index a11030a..0000000 --- a/examples/assets/faithful.csv +++ /dev/null @@ -1,273 +0,0 @@ -"Index", "Eruption length (mins)","Eruption wait (mins)" - 1, 3.600, 79 - 2, 1.800, 54 - 3, 3.333, 74 - 4, 2.283, 62 - 5, 4.533, 85 - 6, 2.883, 55 - 7, 4.700, 88 - 8, 3.600, 85 - 9, 1.950, 51 - 10, 4.350, 85 - 11, 1.833, 54 - 12, 3.917, 84 - 13, 4.200, 78 - 14, 1.750, 47 - 15, 4.700, 83 - 16, 2.167, 52 - 17, 1.750, 62 - 18, 4.800, 84 - 19, 1.600, 52 - 20, 4.250, 79 - 21, 1.800, 51 - 22, 1.750, 47 - 23, 3.450, 78 - 24, 3.067, 69 - 25, 4.533, 74 - 26, 3.600, 83 - 27, 1.967, 55 - 28, 4.083, 76 - 29, 3.850, 78 - 30, 4.433, 79 - 31, 4.300, 73 - 32, 4.467, 77 - 33, 3.367, 66 - 34, 4.033, 80 - 35, 3.833, 74 - 36, 2.017, 52 - 37, 1.867, 48 - 38, 4.833, 80 - 39, 1.833, 59 - 40, 4.783, 90 - 41, 4.350, 80 - 42, 1.883, 58 - 43, 4.567, 84 - 44, 1.750, 58 - 45, 4.533, 73 - 46, 3.317, 83 - 47, 3.833, 64 - 48, 2.100, 53 - 49, 4.633, 82 - 50, 2.000, 59 - 51, 4.800, 75 - 52, 4.716, 90 - 53, 1.833, 54 - 54, 4.833, 80 - 55, 1.733, 54 - 56, 4.883, 83 - 57, 3.717, 71 - 58, 1.667, 64 - 59, 4.567, 77 - 60, 4.317, 81 - 61, 2.233, 59 - 62, 4.500, 84 - 63, 1.750, 48 - 64, 4.800, 82 - 65, 1.817, 60 - 66, 4.400, 92 - 67, 4.167, 78 - 68, 4.700, 78 - 69, 2.067, 65 - 70, 4.700, 73 - 71, 4.033, 82 - 72, 1.967, 56 - 73, 4.500, 79 - 74, 4.000, 71 - 75, 1.983, 62 - 76, 5.067, 76 - 77, 2.017, 60 - 78, 4.567, 78 - 79, 3.883, 76 - 80, 3.600, 83 - 81, 4.133, 75 - 82, 4.333, 82 - 83, 4.100, 70 - 84, 2.633, 65 - 85, 4.067, 73 - 86, 4.933, 88 - 87, 3.950, 76 - 88, 4.517, 80 - 89, 2.167, 48 - 90, 4.000, 86 - 91, 2.200, 60 - 92, 4.333, 90 - 93, 1.867, 50 - 94, 4.817, 78 - 95, 1.833, 63 - 96, 4.300, 72 - 97, 4.667, 84 - 98, 3.750, 75 - 99, 1.867, 51 -100, 4.900, 82 -101, 2.483, 62 -102, 4.367, 88 -103, 2.100, 49 -104, 4.500, 83 -105, 4.050, 81 -106, 1.867, 47 -107, 4.700, 84 -108, 1.783, 52 -109, 4.850, 86 -110, 3.683, 81 -111, 4.733, 75 -112, 2.300, 59 -113, 4.900, 89 -114, 4.417, 79 -115, 1.700, 59 -116, 4.633, 81 -117, 2.317, 50 -118, 4.600, 85 -119, 1.817, 59 -120, 4.417, 87 -121, 2.617, 53 -122, 4.067, 69 -123, 4.250, 77 -124, 1.967, 56 -125, 4.600, 88 -126, 3.767, 81 -127, 1.917, 45 -128, 4.500, 82 -129, 2.267, 55 -130, 4.650, 90 -131, 1.867, 45 -132, 4.167, 83 -133, 2.800, 56 -134, 4.333, 89 -135, 1.833, 46 -136, 4.383, 82 -137, 1.883, 51 -138, 4.933, 86 -139, 2.033, 53 -140, 3.733, 79 -141, 4.233, 81 -142, 2.233, 60 -143, 4.533, 82 -144, 4.817, 77 -145, 4.333, 76 -146, 1.983, 59 -147, 4.633, 80 -148, 2.017, 49 -149, 5.100, 96 -150, 1.800, 53 -151, 5.033, 77 -152, 4.000, 77 -153, 2.400, 65 -154, 4.600, 81 -155, 3.567, 71 -156, 4.000, 70 -157, 4.500, 81 -158, 4.083, 93 -159, 1.800, 53 -160, 3.967, 89 -161, 2.200, 45 -162, 4.150, 86 -163, 2.000, 58 -164, 3.833, 78 -165, 3.500, 66 -166, 4.583, 76 -167, 2.367, 63 -168, 5.000, 88 -169, 1.933, 52 -170, 4.617, 93 -171, 1.917, 49 -172, 2.083, 57 -173, 4.583, 77 -174, 3.333, 68 -175, 4.167, 81 -176, 4.333, 81 -177, 4.500, 73 -178, 2.417, 50 -179, 4.000, 85 -180, 4.167, 74 -181, 1.883, 55 -182, 4.583, 77 -183, 4.250, 83 -184, 3.767, 83 -185, 2.033, 51 -186, 4.433, 78 -187, 4.083, 84 -188, 1.833, 46 -189, 4.417, 83 -190, 2.183, 55 -191, 4.800, 81 -192, 1.833, 57 -193, 4.800, 76 -194, 4.100, 84 -195, 3.966, 77 -196, 4.233, 81 -197, 3.500, 87 -198, 4.366, 77 -199, 2.250, 51 -200, 4.667, 78 -201, 2.100, 60 -202, 4.350, 82 -203, 4.133, 91 -204, 1.867, 53 -205, 4.600, 78 -206, 1.783, 46 -207, 4.367, 77 -208, 3.850, 84 -209, 1.933, 49 -210, 4.500, 83 -211, 2.383, 71 -212, 4.700, 80 -213, 1.867, 49 -214, 3.833, 75 -215, 3.417, 64 -216, 4.233, 76 -217, 2.400, 53 -218, 4.800, 94 -219, 2.000, 55 -220, 4.150, 76 -221, 1.867, 50 -222, 4.267, 82 -223, 1.750, 54 -224, 4.483, 75 -225, 4.000, 78 -226, 4.117, 79 -227, 4.083, 78 -228, 4.267, 78 -229, 3.917, 70 -230, 4.550, 79 -231, 4.083, 70 -232, 2.417, 54 -233, 4.183, 86 -234, 2.217, 50 -235, 4.450, 90 -236, 1.883, 54 -237, 1.850, 54 -238, 4.283, 77 -239, 3.950, 79 -240, 2.333, 64 -241, 4.150, 75 -242, 2.350, 47 -243, 4.933, 86 -244, 2.900, 63 -245, 4.583, 85 -246, 3.833, 82 -247, 2.083, 57 -248, 4.367, 82 -249, 2.133, 67 -250, 4.350, 74 -251, 2.200, 54 -252, 4.450, 83 -253, 3.567, 73 -254, 4.500, 73 -255, 4.150, 88 -256, 3.817, 80 -257, 3.917, 71 -258, 4.450, 83 -259, 2.000, 56 -260, 4.283, 79 -261, 4.767, 78 -262, 4.533, 84 -263, 1.850, 58 -264, 4.250, 83 -265, 1.983, 43 -266, 2.250, 60 -267, 4.750, 75 -268, 4.117, 81 -269, 2.150, 46 -270, 4.417, 90 -271, 1.817, 46 -272, 4.467, 74 diff --git a/examples/fetch.ds b/examples/fetch.ds index ea0e5da..078ec11 100644 --- a/examples/fetch.ds +++ b/examples/fetch.ds @@ -1,4 +1,4 @@ -raw_data = download <"https://api.sampleapis.com/futurama/cast"> -dust_data = from_json +raw_data = (download "https://api.sampleapis.com/futurama/cast") +dust_data = (from_json raw_data) -assert_equal <"Billy West", data.0.name> +(assert_equal "Billy West", dust_data.0.name) diff --git a/src/abstract_tree/function_call.rs b/src/abstract_tree/function_call.rs index 1db5705..fdd7374 100644 --- a/src/abstract_tree/function_call.rs +++ b/src/abstract_tree/function_call.rs @@ -40,7 +40,6 @@ impl AbstractTree for FunctionCall { } else { return Err(Error::FunctionIdentifierNotFound(self.name.clone())); }; - let id_expr_pairs = definition.identifiers().iter().zip(self.arguments.iter()); let mut function_context = context.clone(); diff --git a/src/value/mod.rs b/src/value/mod.rs index 067c8bb..fbc7b41 100644 --- a/src/value/mod.rs +++ b/src/value/mod.rs @@ -249,20 +249,17 @@ impl Sub for Value { type Output = Result; fn sub(self, other: Self) -> Self::Output { - let non_number = match (self, other) { - (Value::Integer(left), Value::Integer(right)) => { - return Ok(Value::Integer(left + right)) - } - (Value::Float(left), Value::Float(right)) => return Ok(Value::Float(left + right)), - (Value::Integer(left), Value::Float(right)) => { - return Ok(Value::Float(left as f64 + right)) - } - (Value::Float(left), Value::Integer(right)) => { - return Ok(Value::Float(left + right as f64)) - } - (non_number, Value::Integer(_)) | (non_number, Value::Float(_)) => non_number, - (non_number, _) => non_number, - }; + match (self.as_int(), other.as_int()) { + (Ok(left), Ok(right)) => return Ok(Value::Integer(left - right)), + _ => {} + } + + match (self.as_number(), other.as_number()) { + (Ok(left), Ok(right)) => return Ok(Value::Float(left - right)), + _ => {} + } + + let non_number = if !self.is_number() { self } else { other }; Err(Error::ExpectedNumber { actual: non_number }) } @@ -295,7 +292,6 @@ impl Div for Value { let left = self.as_number()?; let right = other.as_number()?; let result = left / right; - let value = if result % 2.0 == 0.0 { Value::Integer(result as i64) } else { diff --git a/src/value/variable_map.rs b/src/value/variable_map.rs index bce7b7d..d89e0d5 100644 --- a/src/value/variable_map.rs +++ b/src/value/variable_map.rs @@ -117,7 +117,7 @@ impl VariableMap { } } - /// Removes and assignmed variable. + /// Removes an assigned variable. /// /// TODO: Support dot notation. pub fn remove(&mut self, key: &str) -> Option { diff --git a/tree-sitter-dust/corpus/identifiers.txt b/tree-sitter-dust/corpus/identifiers.txt new file mode 100644 index 0000000..e4242bf --- /dev/null +++ b/tree-sitter-dust/corpus/identifiers.txt @@ -0,0 +1,37 @@ +================== +Simple Identifiers +================== + +x +_y +__xyz__ + +--- + +(root + (item + (statement + (expression + (identifier)))) + (item + (statement + (expression + (identifier)))) + (item + (statement + (expression + (identifier))))) + +================== +Dot Notation +================== + +dust_data.0.name + +--- + +(root + (item + (statement + (expression + (identifier))))) diff --git a/tree-sitter-dust/grammar.js b/tree-sitter-dust/grammar.js index f80875f..2854411 100644 --- a/tree-sitter-dust/grammar.js +++ b/tree-sitter-dust/grammar.js @@ -40,7 +40,7 @@ module.exports = grammar({ $.logic, )), - identifier: $ => /[a-z|_|.]+[0-9]?/, + identifier: $ => /[a-zA-Z|_]+[.a-zA-Z0-9]*/, value: $ => choice( $.integer, diff --git a/tree-sitter-dust/src/grammar.json b/tree-sitter-dust/src/grammar.json index d3777ba..5f65adc 100644 --- a/tree-sitter-dust/src/grammar.json +++ b/tree-sitter-dust/src/grammar.json @@ -147,7 +147,7 @@ }, "identifier": { "type": "PATTERN", - "value": "[a-z|_|.]+[0-9]?" + "value": "[a-zA-Z|_]+[.a-zA-Z0-9]*" }, "value": { "type": "CHOICE", diff --git a/tree-sitter-dust/src/parser.c b/tree-sitter-dust/src/parser.c index 7956b21..2325f2f 100644 --- a/tree-sitter-dust/src/parser.c +++ b/tree-sitter-dust/src/parser.c @@ -1139,15 +1139,15 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == '`') ADVANCE(8); if (lookahead == 'e') ADVANCE(23); if (lookahead == '{') ADVANCE(36); - if (lookahead == '|') ADVANCE(25); + if (lookahead == '|') ADVANCE(21); if (lookahead == '}') ADVANCE(37); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || lookahead == ' ') SKIP(0) if (('0' <= lookahead && lookahead <= '9')) ADVANCE(27); - if (lookahead == '.' || - ('_' <= lookahead && lookahead <= 'z')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('_' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 1: if (lookahead == '!') ADVANCE(6); @@ -1180,15 +1180,15 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == '<') ADVANCE(33); if (lookahead == '=') ADVANCE(7); if (lookahead == '>') ADVANCE(35); - if (lookahead == '|') ADVANCE(25); + if (lookahead == '|') ADVANCE(21); if (lookahead == '}') ADVANCE(37); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || lookahead == ' ') SKIP(2) - if (lookahead == '.' || + if (('A' <= lookahead && lookahead <= 'Z') || lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(26); + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 3: if (lookahead == '"') ADVANCE(29); @@ -1245,15 +1245,15 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == ']') ADVANCE(32); if (lookahead == '`') ADVANCE(8); if (lookahead == '{') ADVANCE(36); - if (lookahead == '|') ADVANCE(25); + if (lookahead == '|') ADVANCE(21); if (lookahead == '}') ADVANCE(37); if (lookahead == '\t' || lookahead == '\n' || lookahead == '\r' || lookahead == ' ') SKIP(13) if (('0' <= lookahead && lookahead <= '9')) ADVANCE(27); - if (lookahead == '.' || - ('_' <= lookahead && lookahead <= 'z')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('_' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 14: if (eof) ADVANCE(16); @@ -1275,8 +1275,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { lookahead == '\r' || lookahead == ' ') SKIP(14) if (('0' <= lookahead && lookahead <= '9')) ADVANCE(27); - if (lookahead == '.' || - ('_' <= lookahead && lookahead <= '|')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('_' <= lookahead && lookahead <= '|')) ADVANCE(25); END_STATE(); case 15: if (eof) ADVANCE(16); @@ -1295,8 +1295,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { lookahead == '\r' || lookahead == ' ') SKIP(15) if (('0' <= lookahead && lookahead <= '9')) ADVANCE(27); - if (lookahead == '.' || - ('_' <= lookahead && lookahead <= '|')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('_' <= lookahead && lookahead <= '|')) ADVANCE(25); END_STATE(); case 16: ACCEPT_TOKEN(ts_builtin_sym_end); @@ -1320,49 +1320,58 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 21: ACCEPT_TOKEN(sym_identifier); + if (lookahead == '_') ADVANCE(25); + if (lookahead == '|') ADVANCE(51); + if (lookahead == '.' || + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 22: ACCEPT_TOKEN(sym_identifier); if (lookahead == 'e') ADVANCE(57); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 23: ACCEPT_TOKEN(sym_identifier); if (lookahead == 'l') ADVANCE(24); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 24: ACCEPT_TOKEN(sym_identifier); if (lookahead == 's') ADVANCE(22); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 25: ACCEPT_TOKEN(sym_identifier); - if (lookahead == '|') ADVANCE(51); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z')) ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 26: ACCEPT_TOKEN(sym_identifier); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(26); END_STATE(); case 27: ACCEPT_TOKEN(sym_integer); @@ -1448,11 +1457,12 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 51: ACCEPT_TOKEN(anon_sym_PIPE_PIPE); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); case 52: ACCEPT_TOKEN(anon_sym_GT_EQ); @@ -1472,11 +1482,12 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 57: ACCEPT_TOKEN(anon_sym_else); if (lookahead == ' ') ADVANCE(10); - if (('0' <= lookahead && lookahead <= '9')) ADVANCE(21); + if (lookahead == '_' || + lookahead == '|') ADVANCE(25); if (lookahead == '.' || - lookahead == '_' || - ('a' <= lookahead && lookahead <= 'z') || - lookahead == '|') ADVANCE(26); + ('0' <= lookahead && lookahead <= '9')) ADVANCE(26); + if (('A' <= lookahead && lookahead <= 'Z') || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(25); END_STATE(); default: return false;