Support character classes

Author: milseman

Sources/Regex2BNF/Regex2BNF.swift

@@ -60,14 +60,26 @@ struct Regex2BNF: ParsableCommand {
 
 //      print("Grouping")
       try convert("a(b|c)d")
+      try convert("a(?:b|c)d")
       try convert("a(bcd|def(g|h)+)z")
 
 //      print("Dot")
-//      try convert(".*")
-//      try convert("(a|b)*.{3}(a|b)")
+      try convert(".*")
+      try convert("(a|b)*.{3}(a|b)")
+
+//      print("Bultin character classes")
+      try convert(#"\(\d{3}\)\d{3}-\d{4}"#)
+      try convert(#"\s+keyword\s+"#)
 
 
 //      print("[Done]")
+
+      // Look at optimizer output, the quant child is very long
+      try convert("a(123456789)+b")
+
+      try convert("Hi the time right now is (AM|PM)")
+
+      try convert("a(b|c)*d{2,4}e?")
     }
     try convert(pattern)
 

Sources/_RegexParser/Regex/BNF/BNF.swift

@@ -1,4 +1,11 @@
-struct BNF {
+protocol BNFNode: CustomStringConvertible {
+  func render() -> String
+}
+extension BNFNode {
+  var description: String { render() }
+}
+
+struct BNF: BNFNode {
   var root: Rule
   var rules: [Rule]
 
@@ -15,7 +22,7 @@ struct BNF {
   }
 }
 
-struct Rule {
+struct Rule: BNFNode {
   // The left-hand side
   var symbol: NonTerminalSymbol
 
@@ -33,23 +40,23 @@ struct CharacterPredicate {
   let impl: (Unicode.Scalar) -> Bool
 }
 
-struct NonTerminalSymbol: Hashable {
+struct NonTerminalSymbol: Hashable, BNFNode {
   var name: String
 
   func render() -> String {
-    name
+    "<\(name)>"
   }
 }
 
-struct Expression {
+struct Expression: BNFNode {
   var choices: [Choice]
 
   func render() -> String {
     "\(choices.map({ $0.render() }).joined(separator: " | "))"
   }
 }
 
-struct Choice {
+struct Choice: BNFNode {
   var sequence: [Symbol]
 
   init(_ symbols: Array<Symbol>) {
@@ -64,29 +71,64 @@ struct Choice {
   }
 }
 
-enum Symbol {
+enum Symbol: BNFNode {
   case terminal(TerminalSymbol)
   case terminalSequence([TerminalSymbol])
   case nonTerminal(NonTerminalSymbol)
+  case builtin(Builtin)
 
   func render() -> String {
     switch self {
     case .terminal(let t):
       return t.render()
+
     case .terminalSequence(let s):
       guard !s.isEmpty else {
         return "\"\""
       }
       return "\(s.map({ $0.render() }).joined(separator: " "))"
+
     case .nonTerminal(let n):
       return n.render()
+
+    case .builtin(let b):
+      return b.render()
+    }
+  }
+}
+
+enum Builtin: BNFNode {
+  case any   // NOTE: we map dot to this, not sure if we want non-newline dots
+  case whitespace
+  case notWhitespace
+  case decimalDigit
+  case notDecimalDigit
+  case wordCharacter
+  case notWordCharacter
+
+  func render() -> String {
+    switch self {
+    case .any:
+      return "<ALL_CHARACTERS_EXCEPT_QUOTE_AND_BACKSLASH>"
+    case .whitespace:
+      return "<WHITESPACES_AND_NEWLINES>"
+    case .notWhitespace:
+      fatalError()
+    case .decimalDigit:
+      return "<DECIMAL_DIGITS>"
+    case .notDecimalDigit:
+      fatalError()
+    case .wordCharacter:
+      return "<ALPHANUMERICS>"
+    case .notWordCharacter:
+      fatalError()
     }
   }
 }
 
 enum CharacterSet {}
 
-enum TerminalSymbol {
+enum TerminalSymbol: BNFNode {
   case character(Unicode.Scalar)
   case characterSet(CharacterSet)
   case utf8CodeUnit(UInt8)

Sources/_RegexParser/Regex/BNF/BNFConvert.swift

@@ -157,13 +157,20 @@ extension BNFConvert {
 
         return [s]
 
+      case .dot:
+        return [.builtin(.any)]
+
+      case .escaped(let b):
+        let builtin = try mapEscapedBuiltin(b)
+        return [.builtin(builtin)]
+
       case .scalar(_): fatalError()
       case .scalarSequence(_): fatalError()
       case .keyboardControl(_): fatalError()
       case .keyboardMeta(_): fatalError()
       case .keyboardMetaControl(_): fatalError()
 
-      case .property, .escaped, .dot, .caretAnchor, .dollarAnchor,
+      case .property, .escaped, .caretAnchor, .dollarAnchor,
           .backreference, .subpattern, .namedCharacter, .callout,
           .backtrackingDirective, .changeMatchingOptions, .invalid:
         fatalError()
@@ -192,6 +199,40 @@ extension BNFConvert {
 }
 
 extension BNFConvert {
+  func mapEscapedBuiltin(_ b: AST.Atom.EscapedBuiltin) throws -> Builtin {
+    switch b {
+
+      // Scalar escapes
+    case .alarm, .escape, .formfeed, .newline, .carriageReturn, .tab, .backspace:
+      fatalError()
+
+      // Built-in character classes
+    case .whitespace: return .whitespace
+    case .notWhitespace: return .notWhitespace
+    case .decimalDigit: return .decimalDigit
+    case .notDecimalDigit: return .notDecimalDigit
+    case .wordCharacter: return .wordCharacter
+    case .notWordCharacter: return .notWordCharacter
+
+      // Other character classes
+    case .horizontalWhitespace, .notHorizontalWhitespace, .notNewline, .newlineSequence, .verticalTab, .notVerticalTab:
+      fatalError()
+
+
+      // Assertions
+    case .wordBoundary, .notWordBoundary:
+      fatalError()
+
+      // Anchors
+    case .startOfSubject, .endOfSubjectBeforeNewline, .endOfSubject, .firstMatchingPositionInSubject:
+      fatalError()
+
+      // Other
+    case .singleDataUnit, .graphemeCluster, .resetStartOfMatch, .trueAnychar, .textSegment, .notTextSegment:
+      fatalError()
+
+    }
+  }
 
   mutating func createQuantify(
     _ child: NonTerminalSymbol,
@@ -232,7 +273,7 @@ extension BNFConvert {
 
     case .zeroOrOne:
       // QUANT ::= QUANT_CHILD | <empty>
-      let name = symbols.genSym("QUANT_+")
+      let name = symbols.genSym("QUANT_?")
       let choices = [
         Choice(child),
         emptyChoice
@@ -413,9 +454,12 @@ extension BNFConvert {
 
       // TODO: This isn't a win when RHS already has uses
       if val.count == 1 {
-        if case .nonTerminal(let rhs) = val.first!.sequence.first! {
-          productions[rootSymbol] = productions[rhs]
-          changed = true
+        let seq = val.first!.sequence
+        if seq.count == 1 {
+          if case .nonTerminal(let rhs) = seq.first! {
+            productions[rootSymbol] = productions[rhs]
+            changed = true
+          }
         }
       }
     }