fix: support hexadecimal numbers and words with underscores

Author: dsherret

src/parse_to_ast.rs

@@ -573,4 +573,24 @@ mod tests {
   fn error_correct_line_column_unicode_width() {
     assert_has_strict_error(r#"["🧑‍🦰", ["#, "Unterminated array on line 1 column 10");
   }
+
+  #[test]
+  fn it_should_parse_unquoted_keys_with_hex_and_trailing_comma() {
+    let parse_result = parse_to_ast(
+      r#"{
+      CP_CanFuncReqId: 0x7DF,  // 2015
+  }"#,
+      &Default::default(),
+      &Default::default(),
+    )
+    .unwrap();
+
+    let value = parse_result.value.unwrap();
+    let obj = value.as_object().unwrap();
+    assert_eq!(obj.properties.len(), 1);
+    assert_eq!(obj.properties[0].name.as_str(), "CP_CanFuncReqId");
+
+    let number_value = obj.properties[0].value.as_number_lit().unwrap();
+    assert_eq!(number_value.value, "0x7DF");
+  }
 }

src/scanner.rs

@@ -160,6 +160,23 @@ impl<'a> Scanner<'a> {
 
     if self.is_zero() {
       self.move_next_char();
+
+      // check for hexadecimal literal (0x or 0X)
+      if matches!(self.current_char(), Some('x') | Some('X')) {
+        self.move_next_char();
+
+        // must have at least one hex digit
+        if !self.is_hex_digit() {
+          return Err(self.create_error_for_current_char(ParseErrorKind::ExpectedDigit));
+        }
+
+        while self.is_hex_digit() {
+          self.move_next_char();
+        }
+
+        let end_byte_index = self.byte_index;
+        return Ok(Token::Number(&self.file_text[start_byte_index..end_byte_index]));
+      }
     } else if self.is_one_nine() {
       self.move_next_char();
       while self.is_digit() {
@@ -288,10 +305,12 @@ impl<'a> Scanner<'a> {
     let start_byte_index = self.byte_index;
 
     while let Some(current_char) = self.current_char() {
-      if current_char.is_whitespace() || current_char == '\r' || current_char == '\n' || current_char == ':' {
+      // check for word terminators
+      if current_char.is_whitespace() || current_char == ':' {
         break;
       }
-      if !current_char.is_alphanumeric() && current_char != '-' {
+      // validate that the character is allowed in a word literal
+      if !current_char.is_alphanumeric() && current_char != '-' && current_char != '_' {
         return Err(self.create_error_for_current_token(ParseErrorKind::UnexpectedToken));
       }
 
@@ -382,6 +401,13 @@ impl<'a> Scanner<'a> {
     self.is_one_nine() || self.is_zero()
   }
 
+  fn is_hex_digit(&self) -> bool {
+    match self.current_char() {
+      Some(current_char) => current_char.is_ascii_hexdigit(),
+      _ => false,
+    }
+  }
+
   fn is_zero(&self) -> bool {
     self.current_char() == Some('0')
   }
@@ -496,6 +522,24 @@ mod tests {
     );
   }
 
+  #[test]
+  fn it_tokenizes_hexadecimal_numbers() {
+    assert_has_tokens(
+      "0x7DF, 0xFF, 0x123ABC, 0xabc, 0X1F",
+      vec![
+        Token::Number("0x7DF"),
+        Token::Comma,
+        Token::Number("0xFF"),
+        Token::Comma,
+        Token::Number("0x123ABC"),
+        Token::Comma,
+        Token::Number("0xabc"),
+        Token::Comma,
+        Token::Number("0X1F"),
+      ],
+    );
+  }
+
   #[test]
   fn it_errors_invalid_exponent() {
     assert_has_error(