Remove most consumer functions (#660) (#733)

This is based heavily off the work in #590, rebased onto main, with
some changes to remove even more consumer uses. Consumer functions
only have two remaining uses: non-ASCII ranges and Unicode lookups
(for things like general category, binary properties, name, etc.).

This change primarily treats custom character classes as alternations
around their contents, with set operations emitted as instructions
instead of implemented via consumer function.

Author: natecook1000

Sources/_StringProcessing/ByteCodeGen.swift

@@ -256,9 +256,11 @@ fileprivate extension Compiler.ByteCodeGen {
     }
   }
 
-  mutating func emitAlternation(
-    _ children: [DSLTree.Node]
-  ) throws {
+  mutating func emitAlternationGen<C: BidirectionalCollection>(
+    _ elements: C,
+    withBacktracking: Bool,
+    _ body: (inout Compiler.ByteCodeGen, C.Element) throws -> Void
+  ) rethrows {
     // Alternation: p0 | p1 | ... | pn
     //     save next_p1
     //     <code for p0>
@@ -276,16 +278,27 @@ fileprivate extension Compiler.ByteCodeGen {
     //     <code for pn>
     //   done:
     let done = builder.makeAddress()
-    for component in children.dropLast() {
+    for element in elements.dropLast() {
       let next = builder.makeAddress()
       builder.buildSave(next)
-      try emitNode(component)
+      try body(&self, element)
+      if !withBacktracking {
+        builder.buildClear()
+      }
       builder.buildBranch(to: done)
       builder.label(next)
     }
-    try emitNode(children.last!)
+    try body(&self, elements.last!)
     builder.label(done)
   }
+  
+  mutating func emitAlternation(
+    _ children: [DSLTree.Node]
+  ) throws {
+    try emitAlternationGen(children, withBacktracking: true) {
+      try $0.emitNode($1)
+    }
+  }
 
   mutating func emitConcatenationComponent(
     _ node: DSLTree.Node
@@ -846,19 +859,187 @@ fileprivate extension Compiler.ByteCodeGen {
       }
   }
 
+  /// Flatten quoted strings into sequences of atoms, so that the standard
+  /// CCC codegen will handle them.
+  func flatteningCustomCharacterClassMembers(
+    _ members: [DSLTree.CustomCharacterClass.Member]
+  ) -> [DSLTree.CustomCharacterClass.Member] {
+    var characters: Set<Character> = []
+    var scalars: Set<UnicodeScalar> = []
+    var result: [DSLTree.CustomCharacterClass.Member] = []
+    for member in members {
+      switch member {
+      case .atom(let atom):
+        switch atom {
+        case let .char(char):
+          characters.insert(char)
+        case let .scalar(scalar):
+          scalars.insert(scalar)
+        default:
+          result.append(member)
+        }
+      case let .quotedLiteral(str):
+        characters.formUnion(str)
+      default:
+        result.append(member)
+      }
+    }
+    result.append(contentsOf: characters.map { .atom(.char($0)) })
+    result.append(contentsOf: scalars.map { .atom(.scalar($0)) })
+    return result
+  }
+  
   func coalescingCustomCharacterClass(
     _ ccc: DSLTree.CustomCharacterClass
   ) -> DSLTree.CustomCharacterClass {
     // This only needs to be done in grapheme semantic mode. In scalar semantic
     // mode, we don't want to coalesce any scalars into a grapheme. This
     // means that e.g `[e\u{301}-\u{302}]` remains a range between U+301 and
     // U+302.
-    guard options.semanticLevel == .graphemeCluster else { return ccc }
-
-    let members = coalescingCustomCharacterClassMembers(ccc.members)
-    return .init(members: members, isInverted: ccc.isInverted)
+    let members = options.semanticLevel == .graphemeCluster
+      ? coalescingCustomCharacterClassMembers(ccc.members)
+      : ccc.members
+    return .init(
+      members: flatteningCustomCharacterClassMembers(members),
+      isInverted: ccc.isInverted)
   }
 
+  mutating func emitCharacterInCCC(_ c: Character)  {
+    switch options.semanticLevel {
+    case .graphemeCluster:
+      emitCharacter(c)
+    case .unicodeScalar:
+      // When in scalar mode, act like an alternation of the individual scalars
+      // that comprise a character.
+      emitAlternationGen(c.unicodeScalars, withBacktracking: false) {
+        $0.emitMatchScalar($1)
+      }
+    }
+  }
+  
+  mutating func emitCCCMember(
+    _ member: DSLTree.CustomCharacterClass.Member
+  ) throws {
+    switch member {
+    case .atom(let atom):
+      switch atom {
+      case .char(let c):
+        emitCharacterInCCC(c)
+      case .scalar(let s):
+        emitCharacterInCCC(Character(s))
+      default:
+        try emitAtom(atom)
+      }
+    case .custom(let ccc):
+      try emitCustomCharacterClass(ccc)
+    case .quotedLiteral:
+      fatalError("Removed in 'flatteningCustomCharacterClassMembers'")
+    case .range:
+      let consumer = try member.generateConsumer(options)
+      builder.buildConsume(by: consumer)
+    case .trivia:
+      return
+      
+    // TODO: Can we decide when it's better to try `rhs` first?
+    // Intersection is trivial, since failure on either side propagates:
+    // - store current position
+    // - lhs
+    // - restore current position
+    // - rhs
+    case let .intersection(lhs, rhs):
+      let r = builder.makePositionRegister()
+      builder.buildMoveCurrentPosition(into: r)
+      try emitCustomCharacterClass(lhs)
+      builder.buildRestorePosition(from: r)
+      try emitCustomCharacterClass(rhs)
+      
+    // TODO: Can we decide when it's better to try `rhs` first?
+    // For subtraction, failure in `lhs` propagates, while failure in `rhs` is
+    // swallowed/reversed:
+    // - store current position
+    // - lhs
+    // - save to end
+    // - restore current position
+    // - rhs
+    // - clear, fail (since both succeeded)
+    // - end: ...
+    case let .subtraction(lhs, rhs):
+      let r = builder.makePositionRegister()
+      let end = builder.makeAddress()
+      builder.buildMoveCurrentPosition(into: r)
+      try emitCustomCharacterClass(lhs)   // no match here = failure, propagates
+      builder.buildSave(end)
+      builder.buildRestorePosition(from: r)
+      try emitCustomCharacterClass(rhs)   // no match here = success, resumes at 'end'
+      builder.buildClear()                // clears 'end'
+      builder.buildFail()                 // this failure propagates outward
+      builder.label(end)
+    
+    // Symmetric difference always requires executing both `rhs` and `lhs`.
+    // Execute each, ignoring failure and storing the resulting position in a
+    // register. If those results are equal, fail. If they're different, use
+    // the position that is different from the starting position:
+    // - store current position as r0
+    // - save to lhsFail
+    // - lhs
+    // - clear lhsFail (and continue)
+    // - lhsFail: save position as r1
+    //
+    // - restore current position
+    // - save to rhsFail
+    // - rhs
+    // - clear rhsFail (and continue)
+    // - rhsFail: save position as r2
+    //
+    // - restore to resulting position from lhs (r1)
+    // - if equal to r2, goto fail (both sides had same result)
+    // - if equal to r0, goto advance (lhs failed)
+    // - goto end
+    // - advance: restore to resulting position from rhs (r2)
+    // - goto end
+    // - fail: fail
+    // - end: ...
+    case let .symmetricDifference(lhs, rhs):
+      let r0 = builder.makePositionRegister()
+      let r1 = builder.makePositionRegister()
+      let r2 = builder.makePositionRegister()
+      let lhsFail = builder.makeAddress()
+      let rhsFail = builder.makeAddress()
+      let advance = builder.makeAddress()
+      let fail = builder.makeAddress()
+      let end = builder.makeAddress()
+
+      builder.buildMoveCurrentPosition(into: r0)
+      builder.buildSave(lhsFail)
+      try emitCustomCharacterClass(lhs)
+      builder.buildClear()
+      builder.label(lhsFail)
+      builder.buildMoveCurrentPosition(into: r1)
+      
+      builder.buildRestorePosition(from: r0)
+      builder.buildSave(rhsFail)
+      try emitCustomCharacterClass(rhs)
+      builder.buildClear()
+      builder.label(rhsFail)
+      builder.buildMoveCurrentPosition(into: r2)
+      
+      // If r1 == r2, then fail
+      builder.buildRestorePosition(from: r1)
+      builder.buildCondBranch(to: fail, ifSamePositionAs: r2)
+      
+      // If r1 == r0, then move to r2 before ending
+      builder.buildCondBranch(to: advance, ifSamePositionAs: r0)
+      builder.buildBranch(to: end)
+      builder.label(advance)
+      builder.buildRestorePosition(from: r2)
+      builder.buildBranch(to: end)
+
+      builder.label(fail)
+      builder.buildFail()
+      builder.label(end)      
+    }
+  }
+  
   mutating func emitCustomCharacterClass(
     _ ccc: DSLTree.CustomCharacterClass
   ) throws {
@@ -876,8 +1057,67 @@ fileprivate extension Compiler.ByteCodeGen {
       }
       return
     }
-    let consumer = try ccc.generateConsumer(options)
-    builder.buildConsume(by: consumer)
+
+    let updatedCCC: DSLTree.CustomCharacterClass
+    if optimizationsEnabled {
+      updatedCCC = ccc.coalescingASCIIMembers(options)
+    } else {
+      updatedCCC = ccc
+    }
+    let filteredMembers = updatedCCC.members.filter({!$0.isOnlyTrivia})
+    
+    if updatedCCC.isInverted {
+      // inverted
+      // custom character class: p0 | p1 | ... | pn
+      // Try each member to make sure they all fail
+      //     save next_p1
+      //     <code for p0>
+      //     clear, fail
+      //   next_p1:
+      //     save next_p2
+      //     <code for p1>
+      //     clear fail
+      //   next_p2:
+      //     save next_p...
+      //     <code for p2>
+      //     clear fail
+      //   ...
+      //   next_pn:
+      //     save done
+      //     <code for pn>
+      //     clear fail
+      //   done:
+      //     step forward by 1
+      let done = builder.makeAddress()
+      for member in filteredMembers.dropLast() {
+        let next = builder.makeAddress()
+        builder.buildSave(next)
+        try emitCCCMember(member)
+        builder.buildClear()
+        builder.buildFail()
+        builder.label(next)
+      }
+      builder.buildSave(done)
+      try emitCCCMember(filteredMembers.last!)
+      builder.buildClear()
+      builder.buildFail()
+      builder.label(done)
+      
+      // Consume a single unit for the inverted ccc
+      switch options.semanticLevel {
+      case .graphemeCluster:
+        builder.buildAdvance(1)
+      case .unicodeScalar:
+        builder.buildAdvanceUnicodeScalar(1)
+      }
+      return
+    }
+    // non inverted CCC
+    // Custom character class: p0 | p1 | ... | pn
+    // Very similar to alternation, but we don't keep backtracking save points
+    try emitAlternationGen(filteredMembers, withBacktracking: false) {
+      try $0.emitCCCMember($1)
+    }
   }
 
   mutating func emitConcatenation(_ children: [DSLTree.Node]) throws {
@@ -1014,6 +1254,12 @@ fileprivate extension Compiler.ByteCodeGen {
 }
 
 extension DSLTree.Node {
+  /// A Boolean value indicating whether this node advances the match position
+  /// on a successful match.
+  ///
+  /// For example, an alternation like `(a|b|c)` always advances the position
+  /// by a character, but `(a|b|)` has an empty branch, which matches without
+  /// advancing.
   var guaranteesForwardProgress: Bool {
     switch self {
     case .orderedChoice(let children):
@@ -1044,12 +1290,34 @@ extension DSLTree.Node {
     case .consumer, .matcher:
       // Allow zero width consumers and matchers
      return false
-    case .customCharacterClass:
-      return true
+    case .customCharacterClass(let ccc):
+      return ccc.guaranteesForwardProgress
     case .quantification(let amount, _, let child):
       let (atLeast, _) = amount.ast.bounds
       return atLeast ?? 0 > 0 && child.guaranteesForwardProgress
     default: return false
     }
   }
 }
+
+extension DSLTree.CustomCharacterClass {
+  /// We allow trivia into CustomCharacterClass, which could result in a CCC
+  /// that matches nothing, ie `(?x)[ ]`.
+  var guaranteesForwardProgress: Bool {
+    for m in members {
+      switch m {
+      case .trivia:
+        continue
+      case let .intersection(lhs, rhs):
+        return lhs.guaranteesForwardProgress && rhs.guaranteesForwardProgress
+      case let .subtraction(lhs, _):
+        return lhs.guaranteesForwardProgress
+      case let .symmetricDifference(lhs, rhs):
+        return lhs.guaranteesForwardProgress && rhs.guaranteesForwardProgress
+      default:
+        return true
+      }
+    }
+    return false
+  }
+}