jeff/dust
1
0
Fork 0
Code Issues

Overhaul integer, float and range parsing

Browse Source
This commit is contained in:
Jeff 2024-01-31 12:10:32 -05:00
parent 65afe9dd29
commit 820c863f7f
11 changed files with 26387 additions and 25030 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/abstract_tree/for.rs
View File

@ -66,7 +66,7 @@ impl AbstractTree for For {
} else { } else {
let loop_context = Map::clone_from(context)?; let loop_context = Map::clone_from(context)?;
for i in range.start..range.end { for i in range {
loop_context.set(key.clone(), Value::Integer(i))?; loop_context.set(key.clone(), Value::Integer(i))?;
self.block.run(source, &loop_context)?; self.block.run(source, &loop_context)?;

18
src/abstract_tree/value_node.rs
View File

@ -1,4 +1,8 @@
use std::{cmp::Ordering, collections::BTreeMap, ops::Range}; use std::{
cmp::Ordering,
collections::BTreeMap,
ops::{Range, RangeInclusive},
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -19,7 +23,7 @@ pub enum ValueNode {
Map(BTreeMap<String, (Statement, Option<Type>)>), Map(BTreeMap<String, (Statement, Option<Type>)>),
BuiltInValue(BuiltInValue), BuiltInValue(BuiltInValue),
Structure(BTreeMap<String, (Option<Statement>, Type)>), Structure(BTreeMap<String, (Option<Statement>, Type)>),
Range(Range<i64>), Range(RangeInclusive<i64>),
} }
impl AbstractTree for ValueNode { impl AbstractTree for ValueNode {
@ -164,13 +168,11 @@ impl AbstractTree for ValueNode {
ValueNode::Structure(btree_map) ValueNode::Structure(btree_map)
} }
"range" => { "range" => {
let start_node = child.child(0).unwrap(); let mut split = source[child.byte_range()].split("..");
let end_node = child.child(2).unwrap(); let start = split.next().unwrap().parse().unwrap();
let end = split.next().unwrap().parse().unwrap();
let start = source[start_node.byte_range()].parse().unwrap(); ValueNode::Range(start..=end)
let end = source[end_node.byte_range()].parse().unwrap();
ValueNode::Range(Range { start, end })
} }
_ => { _ => {
return Err(Error::UnexpectedSyntaxNode { return Err(Error::UnexpectedSyntaxNode {

20
src/value/mod.rs
View File

@ -15,7 +15,7 @@ use std::{
convert::TryFrom, convert::TryFrom,
fmt::{self, Display, Formatter}, fmt::{self, Display, Formatter},
marker::PhantomData, marker::PhantomData,
ops::{Add, AddAssign, Div, Mul, Range, Rem, Sub, SubAssign}, ops::{Add, AddAssign, Div, Mul, Range, RangeInclusive, Rem, Sub, SubAssign},
}; };
pub use self::{function::Function, list::List, map::Map, structure::Structure}; pub use self::{function::Function, list::List, map::Map, structure::Structure};
@ -39,7 +39,7 @@ pub enum Value {
Float(f64), Float(f64),
Integer(i64), Integer(i64),
Boolean(bool), Boolean(bool),
Range(Range<i64>), Range(RangeInclusive<i64>),
Option(Option<Box<Value>>), Option(Option<Box<Value>>),
Structure(Structure), Structure(Structure),
} }
@ -56,7 +56,7 @@ impl Value {
} }
pub fn range(start: i64, end: i64) -> Self { pub fn range(start: i64, end: i64) -> Self {
Value::Range(Range { start, end }) Value::Range(start..=end)
} }
pub fn r#type(&self) -> Type { pub fn r#type(&self) -> Type {
@ -296,7 +296,9 @@ impl Add for Value {
fn add(self, other: Self) -> Self::Output { fn add(self, other: Self) -> Self::Output {
if let (Ok(left), Ok(right)) = (self.as_integer(), other.as_integer()) { if let (Ok(left), Ok(right)) = (self.as_integer(), other.as_integer()) {
return Ok(Value::Integer(left + right)); let (sum, _) = left.overflowing_add(right);
return Ok(Value::Integer(sum));
} }
if let (Ok(left), Ok(right)) = (self.as_number(), other.as_number()) { if let (Ok(left), Ok(right)) = (self.as_number(), other.as_number()) {
@ -328,7 +330,9 @@ impl Sub for Value {
fn sub(self, other: Self) -> Self::Output { fn sub(self, other: Self) -> Self::Output {
if let (Ok(left), Ok(right)) = (self.as_integer(), other.as_integer()) { if let (Ok(left), Ok(right)) = (self.as_integer(), other.as_integer()) {
return Ok(Value::Integer(left - right)); let (difference, _) = left.overflowing_sub(right);
return Ok(Value::Integer(difference));
} }
if let (Ok(left), Ok(right)) = (self.as_number(), other.as_number()) { if let (Ok(left), Ok(right)) = (self.as_number(), other.as_number()) {
@ -479,8 +483,8 @@ impl Ord for Value {
(Value::Structure(left), Value::Structure(right)) => left.cmp(right), (Value::Structure(left), Value::Structure(right)) => left.cmp(right),
(Value::Structure(_), _) => Ordering::Greater, (Value::Structure(_), _) => Ordering::Greater,
(Value::Range(left), Value::Range(right)) => { (Value::Range(left), Value::Range(right)) => {
let left_len = left.end - left.start; let left_len = left.end() - left.start();
let right_len = right.end - right.start; let right_len = right.end() - right.start();
left_len.cmp(&right_len) left_len.cmp(&right_len)
} }
@ -538,7 +542,7 @@ impl Display for Value {
Value::Map(map) => write!(f, "{map}"), Value::Map(map) => write!(f, "{map}"),
Value::Function(function) => write!(f, "{function}"), Value::Function(function) => write!(f, "{function}"),
Value::Structure(structure) => write!(f, "{structure}"), Value::Structure(structure) => write!(f, "{structure}"),
Value::Range(range) => write!(f, "{}..{}", range.start, range.end), Value::Range(range) => write!(f, "{}..{}", range.start(), range.end()),
} }
} }
} }

5
tests/value.rs
View File

@ -27,13 +27,12 @@ fn integer_overflow() {
#[test] #[test]
fn float() { fn float() {
assert_eq!(interpret("0.1"), Ok(Value::Float(0.1)));
assert_eq!( assert_eq!(
interpret("1.7976931348623157e308f64"), interpret("1.7976931348623157e308"),
Ok(Value::Float(f64::MAX)) Ok(Value::Float(f64::MAX))
); );
assert_eq!( assert_eq!(
interpret("-1.7976931348623157e308f64"), interpret("-1.7976931348623157e308"),
Ok(Value::Float(f64::MIN)) Ok(Value::Float(f64::MIN))
); );
} }

4
tree-sitter-dust/corpus/index.txt
View File

@ -85,9 +85,7 @@ Nested Indexes
(value (value
(integer))))) (integer)))))
(index_expression (index_expression
(range (range))))))
(integer)
(integer)))))))
================================================================================ ================================================================================
Function Call Index Function Call Index

4
tree-sitter-dust/corpus/range.txt
View File

@ -10,6 +10,4 @@ Simple Range
(statement (statement
(expression (expression
(value (value
(range (range)))))
(integer)
(integer))))))

60
tree-sitter-dust/corpus/values.txt
View File

@ -22,7 +22,7 @@ Integers
================================================================================ ================================================================================
1 2 3 1 2 3
456 7 -456 -7
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
@ -48,6 +48,64 @@ Integers
(value (value
(integer))))) (integer)))))
================================================================================
Floats
================================================================================
0.0
1.0
666.0
-0.666
NaN
Infinity
infinity
nan
nana
infinity_gauntlet
--------------------------------------------------------------------------------
(root
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(value
(float))))
(statement
(expression
(identifier)))
(statement
(expression
(identifier))))
================================================================================ ================================================================================
Strings Strings
================================================================================ ================================================================================

69
tree-sitter-dust/grammar.js
View File

@ -125,12 +125,7 @@ module.exports = grammar({
$.range, $.range,
), ),
range: $ => range: $ => /\d+[.]{2}\d+/,
seq(
$.integer,
token.immediate('..'),
$.integer,
),
structure: $ => structure: $ =>
seq( seq(
@ -181,65 +176,15 @@ module.exports = grammar({
'}', '}',
), ),
integer: $ => integer: $ => /[-]?\d+/,
token(
prec.left(
seq(
optional('-'),
repeat1(
choice(
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'0',
),
),
),
),
),
float: $ => float: $ =>
token(
prec.left(
seq(
optional('-'),
repeat1(
choice( choice(
'1', /[-|+]?\d*[.][\d|e|-]*/,
'2', "Infinity",
'3', "infinity",
'4', "NaN",
'5', "nan",
'6',
'7',
'8',
'9',
'0',
),
),
'.',
repeat1(
choice(
'1',
'2',
'3',
'4',
'5',
'6',
'7',
'8',
'9',
'0',
),
),
),
),
), ),
string: $ => string: $ =>

223
tree-sitter-dust/src/grammar.json
View File

@ -352,24 +352,8 @@
] ]
}, },
"range": { "range": {
"type": "SEQ", "type": "PATTERN",
"members": [ "value": "\\d+[.]{2}\\d+"
{
"type": "SYMBOL",
"name": "integer"
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": ".."
}
},
{
"type": "SYMBOL",
"name": "integer"
}
]
}, },
"structure": { "structure": {
"type": "SEQ", "type": "SEQ",
@ -515,201 +499,34 @@
] ]
}, },
"integer": { "integer": {
"type": "TOKEN", "type": "PATTERN",
"content": { "value": "[-]?\\d+"
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "BLANK"
}
]
},
{
"type": "REPEAT1",
"content": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "1"
},
{
"type": "STRING",
"value": "2"
},
{
"type": "STRING",
"value": "3"
},
{
"type": "STRING",
"value": "4"
},
{
"type": "STRING",
"value": "5"
},
{
"type": "STRING",
"value": "6"
},
{
"type": "STRING",
"value": "7"
},
{
"type": "STRING",
"value": "8"
},
{
"type": "STRING",
"value": "9"
},
{
"type": "STRING",
"value": "0"
}
]
}
}
]
}
}
}, },
"float": { "float": {
"type": "TOKEN",
"content": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "STRING", "type": "PATTERN",
"value": "-" "value": "[-|+]?\\d*[.][\\d|e|f|-]*"
}, },
{ {
"type": "BLANK" "type": "STRING",
"value": "Infinity"
},
{
"type": "STRING",
"value": "infinity"
},
{
"type": "STRING",
"value": "NaN"
},
{
"type": "STRING",
"value": "nan"
} }
] ]
}, },
{
"type": "REPEAT1",
"content": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "1"
},
{
"type": "STRING",
"value": "2"
},
{
"type": "STRING",
"value": "3"
},
{
"type": "STRING",
"value": "4"
},
{
"type": "STRING",
"value": "5"
},
{
"type": "STRING",
"value": "6"
},
{
"type": "STRING",
"value": "7"
},
{
"type": "STRING",
"value": "8"
},
{
"type": "STRING",
"value": "9"
},
{
"type": "STRING",
"value": "0"
}
]
}
},
{
"type": "STRING",
"value": "."
},
{
"type": "REPEAT1",
"content": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "1"
},
{
"type": "STRING",
"value": "2"
},
{
"type": "STRING",
"value": "3"
},
{
"type": "STRING",
"value": "4"
},
{
"type": "STRING",
"value": "5"
},
{
"type": "STRING",
"value": "6"
},
{
"type": "STRING",
"value": "7"
},
{
"type": "STRING",
"value": "8"
},
{
"type": "STRING",
"value": "9"
},
{
"type": "STRING",
"value": "0"
}
]
}
}
]
}
}
},
"string": { "string": {
"type": "PATTERN", "type": "PATTERN",
"value": "(\"[^\"]*?\")|('[^']*?')|(`[^`]*?`)" "value": "(\"[^\"]*?\")|('[^']*?')|(`[^`]*?`)"

48
tree-sitter-dust/src/node-types.json
View File

@ -161,6 +161,11 @@
] ]
} }
}, },
{
"type": "float",
"named": true,
"fields": {}
},
{ {
"type": "for", "type": "for",
"named": true, "named": true,
@ -534,21 +539,6 @@
] ]
} }
}, },
{
"type": "range",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "integer",
"named": true
}
]
}
},
{ {
"type": "return", "type": "return",
"named": true, "named": true,
@ -833,10 +823,6 @@
"type": "->", "type": "->",
"named": false "named": false
}, },
{
"type": "..",
"named": false
},
{ {
"type": "/", "type": "/",
"named": false "named": false
@ -877,6 +863,14 @@
"type": ">=", "type": ">=",
"named": false "named": false
}, },
{
"type": "Infinity",
"named": false
},
{
"type": "NaN",
"named": false
},
{ {
"type": "[", "type": "[",
"named": false "named": false
@ -937,10 +931,6 @@
"type": "false", "type": "false",
"named": false "named": false
}, },
{
"type": "float",
"named": true
},
{ {
"type": "float", "type": "float",
"named": false "named": false
@ -965,6 +955,10 @@
"type": "in", "type": "in",
"named": false "named": false
}, },
{
"type": "infinity",
"named": false
},
{ {
"type": "int", "type": "int",
"named": false "named": false
@ -989,6 +983,10 @@
"type": "match", "type": "match",
"named": false "named": false
}, },
{
"type": "nan",
"named": false
},
{ {
"type": "new", "type": "new",
"named": false "named": false
@ -1013,6 +1011,10 @@
"type": "random", "type": "random",
"named": false "named": false
}, },
{
"type": "range",
"named": true
},
{ {
"type": "return", "type": "return",
"named": false "named": false

50948
tree-sitter-dust/src/parser.c
View File

File diff suppressed because it is too large Load Diff