Implement direct AST -> DSLTree conversion

Author: natecook1000

Sources/_StringProcessing/Compiler.swift

@@ -12,21 +12,30 @@
 internal import _RegexParser
 
 class Compiler {
-  let tree: DSLTree
+  let tree: DSLList
 
   // TODO: Or are these stored on the tree?
   var options = MatchingOptions()
   private var compileOptions: _CompileOptions = .default
 
   init(ast: AST) {
-    self.tree = ast.dslTree
+    self.tree = DSLList(tree: ast.dslTree)
   }
 
   init(tree: DSLTree) {
-    self.tree = tree
+    self.tree = DSLList(tree: tree)
+  }
+
+  init(list: DSLList) {
+    self.tree = list
   }
 
   init(tree: DSLTree, compileOptions: _CompileOptions) {
+    self.tree = DSLList(tree: tree)
+    self.compileOptions = compileOptions
+  }
+
+  init(tree: DSLList, compileOptions: _CompileOptions) {
     self.tree = tree
     self.compileOptions = compileOptions
   }
@@ -42,18 +51,20 @@ class Compiler {
       compileOptions:
         compileOptions,
       captureList: tree.captureList)
-    return try codegen.emitRoot(tree.root)
+    fatalError()
+//    return try codegen.emitRoot(tree.root)
   }
 
   __consuming func emitViaList() throws -> MEProgram {
     // TODO: Handle global options
-    var dslList = DSLList(tree: tree)
+//    var dslList = DSLList(tree: tree)
     var codegen = ByteCodeGen(
       options: options,
       compileOptions:
         compileOptions,
       captureList: tree.captureList)
-    return try codegen.emitRoot(&dslList)
+    var tree = tree
+    return try codegen.emitRoot(&tree)
   }
 }
 
@@ -105,20 +116,22 @@ func _compileRegex(
   _ syntax: SyntaxOptions = .traditional,
   _ semanticLevel: RegexSemanticLevel? = nil
 ) throws -> MEProgram {
-  let ast = try parse(regex, syntax)
-  let dsl: DSLTree
+  var ast = try parse(regex, syntax)
+  let dsl: DSLList
 
   switch semanticLevel?.base {
   case .graphemeCluster:
     let sequence = AST.MatchingOptionSequence(adding: [.init(.graphemeClusterSemantics, location: .fake)])
-    dsl = DSLTree(.nonCapturingGroup(.init(ast: .changeMatchingOptions(sequence)), ast.dslTree.root))
+    ast.root = AST.Node.group(AST.Group(.init(faking: .changeMatchingOptions(sequence)), ast.root, .fake))
+    dsl = DSLList(ast: ast)
   case .unicodeScalar:
     let sequence = AST.MatchingOptionSequence(adding: [.init(.unicodeScalarSemantics, location: .fake)])
-    dsl = DSLTree(.nonCapturingGroup(.init(ast: .changeMatchingOptions(sequence)), ast.dslTree.root))
+    ast.root = AST.Node.group(AST.Group(.init(faking: .changeMatchingOptions(sequence)), ast.root, .fake))
+    dsl = DSLList(ast: ast)
   case .none:
-    dsl = ast.dslTree
+    dsl = DSLList(ast: ast)
   }
-  let program = try Compiler(tree: dsl).emit()
+  let program = try Compiler(list: dsl).emit()
   return program
 }
 

Sources/_StringProcessing/LiteralPrinter.swift

@@ -36,7 +36,8 @@ extension Regex {
   @available(SwiftStdlib 6.0, *)
   public var _literalPattern: String? {
     var gen = LiteralPrinter(options: MatchingOptions())
-    gen.outputNode(self.program.tree.root)
+    var list = self.program.tree.nodes[...]
+    try? gen.outputList(&list)
     return gen.canonicalLiteralString
   }
 }
@@ -83,6 +84,159 @@ fileprivate struct LiteralPrinter {
   mutating func saveInconvertible(_ node: DSLTree.Node) {
     segments.append(.inconvertible(node))
   }
+  
+  mutating func inconvertible(_ node: DSLTree.Node) throws {
+    segments.append(.inconvertible(node))
+    throw Incovertible.error
+  }
+}
+
+extension LiteralPrinter {
+  enum Incovertible: Error {
+    case error
+  }
+  
+  mutating func outputList(_ list: inout ArraySlice<DSLTree.Node>) throws {
+    guard let node = list.popFirst() else {
+      return
+    }
+    
+    switch node {
+    case let .orderedChoice(children):
+      try outputAlternation(&list, count: children.count)
+    case let .concatenation(children):
+      try outputConcatenation(&list, count: children.count)
+      
+    case let .capture(name, nil, _, nil):
+      options.beginScope()
+      defer { options.endScope() }
+      try outputCapture(&list, name: name)
+    case .capture:
+      // Captures that use a reference or a transform are unsupported
+      try inconvertible(node)
+      return
+      
+    case let .nonCapturingGroup(kind, _):
+      guard let kindPattern = kind._patternString else {
+        try inconvertible(node)
+        return
+      }
+      options.beginScope()
+      defer { options.endScope() }
+      
+      output(kindPattern)
+      if case .changeMatchingOptions(let optionSequence) = kind.ast {
+        options.apply(optionSequence)
+      }
+      try outputList(&list)
+      output(")")
+      
+    case .ignoreCapturesInTypedOutput(_),
+      .limitCaptureNesting(_):
+      try outputList(&list)
+    case let .quantification(amount, kind, _):
+      try outputQuantification(&list, amount: amount, kind: kind)
+    case let .customCharacterClass(charClass):
+      outputCustomCharacterClass(charClass)
+    case let .atom(atom):
+      outputAtom(atom)
+    case let .quotedLiteral(literal):
+      output(prepareQuotedLiteral(literal))
+      
+    case .trivia(_):
+      // TODO: Include trivia?
+      return
+    case .empty:
+      return
+      
+    case .conditional, .absentFunction, .consumer, .matcher, .characterPredicate:
+      saveInconvertible(node)
+    }
+  }
+  
+  mutating func outputAlternation(_ list: inout ArraySlice<DSLTree.Node>, count: Int) throws {
+    for i in 0..<count {
+      if i != 0 {
+        output("|")
+      }
+      try outputList(&list)
+    }
+  }
+  
+  mutating func outputConcatenation(_ list: inout ArraySlice<DSLTree.Node>, count: Int) throws {
+    for _ in 0..<count {
+      try outputList(&list)
+    }
+  }
+  
+  mutating func outputCapture(_ list: inout ArraySlice<DSLTree.Node>, name: String?) throws {
+    if let name {
+      output("(?<\(name)>")
+    } else {
+      output("(")
+    }
+    try outputList(&list)
+    output(")")
+  }
+  
+  func requiresGrouping(_ list: ArraySlice<DSLTree.Node>) -> Bool {
+    guard let node = list.first else { return false } // malformed?
+    switch node {
+    case .concatenation(let children):
+      switch children.count {
+      case 0:
+        return false
+      case 1:
+        return requiresGrouping(list.dropFirst())
+      default:
+        return true
+      }
+      
+    case .quotedLiteral(let literal):
+      return prepareQuotedLiteral(literal).count > 1
+      
+    default:
+      return false
+    }
+  }
+
+  mutating func outputQuantification(
+    _ list: inout ArraySlice<DSLTree.Node>,
+    amount: DSLTree._AST.QuantificationAmount,
+    kind: DSLTree.QuantificationKind
+  ) throws {
+    // RegexBuilder regexes can have children that need
+    if requiresGrouping(list) {
+      output("(?:")
+      try outputList(&list)
+      output(")")
+    } else {
+      try outputList(&list)
+    }
+
+    switch amount.ast {
+    case .zeroOrMore:
+      output("*")
+    case .oneOrMore:
+      output("+")
+    case .zeroOrOne:
+      output("?")
+    case let .exactly(n):
+      output("{\(n.value!)}")
+    case let .nOrMore(n):
+      output("{\(n.value!),}")
+    case let .upToN(n):
+      output("{,\(n.value!)}")
+    case let .range(low, high):
+      output("{\(low.value!),\(high.value!)}")
+    #if RESILIENT_LIBRARIES
+    @unknown default:
+      fatalError()
+    #endif
+    }
+    
+    outputQuantificationKind(kind)
+  }
 }
 
 extension LiteralPrinter {

Sources/_StringProcessing/Regex/ASTConversion.swift

@@ -17,6 +17,120 @@ extension AST {
   }
 }
 
+extension AST.Node {
+  func convert(into list: inout [DSLTree.Node]) throws {
+    switch self {
+    case .alternation(let alternation):
+      list.append(.orderedChoice(Array(repeating: TEMP_FAKE_NODE, count: alternation.children.count)))
+      for child in alternation.children {
+        try child.convert(into: &list)
+      }
+    case .concatenation(let concatenation):
+      let coalesced = self.coalescedChildren
+      list.append(.concatenation(Array(repeating: TEMP_FAKE_NODE, count: coalesced.count)))
+      for child in coalesced {
+        try child.convert(into: &list)
+      }
+    case .group(let group):
+      let child = group.child
+      switch group.kind.value {
+      case .capture:
+        list.append(.capture(TEMP_FAKE_NODE))
+        try child.convert(into: &list)
+      case .namedCapture(let name):
+        list.append(.capture(name: name.value, TEMP_FAKE_NODE))
+        try child.convert(into: &list)
+      case .balancedCapture:
+        throw Unsupported("TODO: balanced captures")
+      default:
+        list.append(.nonCapturingGroup(.init(ast: group.kind.value), TEMP_FAKE_NODE))
+        try child.convert(into: &list)
+      }
+    case .conditional(let conditional):
+      list.append(.conditional(.init(ast: conditional.condition.kind), TEMP_FAKE_NODE, TEMP_FAKE_NODE))
+      try conditional.trueBranch.convert(into: &list)
+      try conditional.falseBranch.convert(into: &list)
+    case .quantification(let quant):
+      list.append(
+        .quantification(.init(ast: quant.amount.value), .syntax(.init(ast: quant.kind.value)), TEMP_FAKE_NODE))
+      try quant.child.convert(into: &list)
+    case .quote(let node):
+      list.append(.quotedLiteral(node.literal))
+    case .trivia(let node):
+      list.append(.trivia(node.contents))
+    case .interpolation(_):
+      throw Unsupported("TODO: interpolation")
+    case .atom(let atom):
+      switch atom.kind {
+      case .scalarSequence(let seq):
+        // The DSL doesn't have an equivalent node for scalar sequences. Splat
+        // them into a concatenation of scalars.
+        //        list.append(.concatenation(Array(repeating: TEMP_FAKE_NODE, count: seq.scalarValues.count)))
+        list.append(.quotedLiteral(String(seq.scalarValues)))
+      default:
+        list.append(.atom(atom.dslTreeAtom))
+      }
+    case .customCharacterClass(let ccc):
+      list.append(.customCharacterClass(ccc.dslTreeClass))
+    case .absentFunction(let abs):
+      // TODO: What should this map to?
+      list.append(.absentFunction(.init(ast: abs)))
+    case .empty(_):
+      list.append(.empty)
+    }
+  }
+  
+  var coalescedChildren: [AST.Node] {
+    // Before converting a concatenation in a tree to list form, we need to
+    // flatten out any nested concatenations, and coalesce any adjacent
+    // characters and scalars, forming quoted literals of their contents,
+    // over which we can perform grapheme breaking.
+
+    func flatten(_ node: AST.Node) -> [AST.Node] {
+      switch node {
+      case .concatenation(let concat):
+        return concat.children.flatMap(flatten)
+      default:
+        return [node]
+      }
+    }
+    
+    switch self {
+    case .alternation(let v): return v.children
+    case .concatenation(let v):
+      let children = v.children
+        .flatMap(flatten)
+        .coalescing(with: "", into: { AST.Node.quote(.init($0, .fake)) }) { str, node in
+          switch node {
+          case .atom(let a):
+            guard let c = a.literalCharacterValue else { return false }
+            str.append(c)
+            return true
+          case .quote(let q):
+            str += q.literal
+            return true
+          case .trivia:
+            // Trivia can be completely ignored if we've already coalesced
+            // something.
+            return !str.isEmpty
+          default:
+            return false
+          }
+        }
+      return children
+
+    case .group(let group):
+      return [group.child]
+    case .conditional(let conditional):
+      return [conditional.trueBranch, conditional.falseBranch]
+    case .quantification(let quant):
+      return [quant.child]
+    case .quote, .trivia, .interpolation, .atom, .customCharacterClass, .absentFunction, .empty:
+      return []
+    }
+  }
+}
+
 extension AST.Node {
   /// Converts an AST node to a `convertedRegexLiteral` node.
   var dslTreeNode: DSLTree.Node {

Sources/_StringProcessing/Regex/AnyRegexOutput.swift

@@ -284,7 +284,7 @@ extension Regex where Output == AnyRegexOutput {
   /// - Parameter regex: A regular expression to convert to use a dynamic
   ///   capture list.
   public init<OtherOutput>(_ regex: Regex<OtherOutput>) {
-    self.init(node: regex.root)
+    self.init(list: regex.list)
   }
 }
 
@@ -331,7 +331,7 @@ extension Regex {
     _ regex: Regex<AnyRegexOutput>,
     as outputType: Output.Type = Output.self
   ) {
-    self.init(node: regex.root)
+    self.init(list: regex.list)
 
     guard _verifyType().0 else {
       return nil