three-phase validation and styling

Package.resolved

@@ -14,7 +14,7 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/ChimeHQ/SwiftTreeSitter",
       "state" : {
-        "revision" : "10cb68c00a9963c2884b30f168a9de377790d812"
+        "revision" : "7ccb58219c3e928bf1938dd43d33fd349dc71808"
       }
     }
   ],

Package.swift

@@ -15,7 +15,7 @@ let package = Package(
 		.library(name: "Neon", targets: ["Neon"]),
 	],
 	dependencies: [
-		.package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", revision: "10cb68c00a9963c2884b30f168a9de377790d812"),
+		.package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", revision: "7ccb58219c3e928bf1938dd43d33fd349dc71808"),
 		.package(url: "https://github.com/ChimeHQ/Rearrange", from: "1.8.1"),
 	],
 	targets: [

README.md

@@ -52,8 +52,10 @@ Neon's lowest-level component is called RangeState. This module contains the cor
 
 - `Hybrid(Throwing)ValueProvider`: a fundamental type that defines work in terms of both synchronous and asynchronous functions
 - `RangeProcessor`: performs on-demand processing of range-based content (think parsing)
-- `RangeValidator`: manages the validation of range-based content (think highlighting)
+- `RangeValidator`: building block for managing the validation of range-based content
 - `RangeInvalidationBuffer`: buffer and consolidate invalidations so they can be applied at the optimal time
+- `SinglePhaseRangeValidator`: performs validation with a single data source (single-phase highlighting)
+- `ThreePhaseRangeValidator`: performs validation with primary, fallback, and secondary data sources (three-phase highlighting)
 
 Many of these support versionable content. If you are working with a backing store structure that supports efficient versioning, like a [piece table](https://en.wikipedia.org/wiki/Piece_table), expressing this to RangeState can improve its efficiency.
 
@@ -78,6 +80,8 @@ The top-level module includes systems for managing text styling. It is also text
 - `TextViewHighlighter`: simple integration between `NSTextView`/`UITextView` and `TreeSitterClient`
 - `TextViewSystemInterface`: implementation of the `TextSystemInterface` protocol for `NSTextView`/`UITextView`
 - `LayoutManagerSystemInterface`, `TextLayoutManagerSystemInterface`, and `TextStorageSystemInterface`: Specialized TextKit 1/2 implementations `TextSystemInterface`
+- `TextSystemStyler`: a style manager that works with a single `TokenProvider`
+- `ThreePhaseTextSystemStyler`: a true three-phase style manager that combines a primary, fallback and secondary token data sources
 
 There is also an example project that demonstrates how to use `TextViewHighlighter` for macOS and iOS.
 

Sources/Neon/TextSystemInterface+Validation.swift

@@ -0,0 +1,74 @@
+import Foundation
+
+import RangeState
+
+extension TextSystemInterface {
+	typealias Provider = ThreePhaseRangeValidator<Content>.Provider
+	typealias ContentRange = ThreePhaseRangeValidator<Content>.ContentRange
+
+	@MainActor
+	func validation(for application: TokenApplication, in contentRange: ContentRange) -> Validation {
+		let effectiveRange = application.range ?? contentRange.value
+
+		applyStyles(for: application)
+
+		return .success(effectiveRange)
+	}
+
+	@MainActor
+	func asyncValidate(
+		_ contentRange: ContentRange,
+		provider: @MainActor (NSRange) async -> TokenApplication
+	) async -> Validation {
+		guard contentRange.version == content.currentVersion else { return .stale }
+
+		// https://github.com/apple/swift/pull/71143
+		let application = await provider(contentRange.value)
+
+		// second check after the awit
+		guard contentRange.version == content.currentVersion else { return .stale }
+
+		return validation(for: application, in: contentRange)
+	}
+
+	@MainActor
+	func validationProvider(with provider: TokenProvider) -> Provider {
+		.init(
+			syncValue: { contentRange in
+				guard contentRange.version == self.content.currentVersion else { return .stale }
+
+				guard let application = provider.sync(contentRange.value) else {
+					return nil
+				}
+
+				return validation(for: application, in: contentRange)
+			},
+			mainActorAsyncValue: { contentRange in
+				return await asyncValidate(contentRange, provider: provider.mainActorAsync)
+			}
+		)
+	}
+}
+extension TextSystemInterface {
+	typealias Styler = ThreePhaseTextSystemStyler<Self>
+	typealias FallbackHandler = ThreePhaseRangeValidator<Self.Content>.FallbackHandler
+	typealias SecondaryValidationProvider = ThreePhaseRangeValidator<Self.Content>.SecondaryValidationProvider
+
+	@MainActor
+	func validatorFallbackHandler(
+		with provider: @escaping Styler.FallbackTokenProvider
+	) -> FallbackHandler {
+		{ range in
+			let application = provider(range)
+
+			applyStyles(for: application)
+		}
+	}
+
+	@MainActor
+	func validatorSecondaryHandler(
+		with provider: @escaping Styler.SecondaryValidationProvider
+	) -> SecondaryValidationProvider {
+		{ await asyncValidate($0, provider: provider) }
+	}
+}

Sources/Neon/TextSystemStyler.swift

@@ -11,25 +11,24 @@ import RangeState
 /// > Note: A `Styler` must be informed of all text content changes made using `didChangeContent(in:, delta:)` and changes to the content visibility with `contentVisibleRectChanged(_:_`.
 @MainActor
 public final class TextSystemStyler<Interface: TextSystemInterface> {
-	typealias Validator = RangeValidator<Interface.Content>
-
 	private let textSystem: Interface
-	private let validator: Validator
+	private let tokenProvider: TokenProvider
+	private let validator: SinglePhaseRangeValidator<Interface.Content>
 
 	public init(textSystem: Interface, tokenProvider: TokenProvider) {
 		self.textSystem = textSystem
+		self.tokenProvider = tokenProvider
 
 		let tokenValidator = TokenSystemValidator(
 			textSystem: textSystem,
 			tokenProvider: tokenProvider
 		)
 
-		self.validator = Validator(
+		self.validator = SinglePhaseRangeValidator(
 			configuration: .init(
 				versionedContent: textSystem.content,
-				validationProvider: tokenValidator.validationProvider,
-				workingRangeProvider: { textSystem.visibleRange },
-				automatic: true
+				provider: tokenValidator.validationProvider,
+				priorityRangeProvider: { textSystem.visibleRange }
 			)
 		)
 	}
@@ -56,15 +55,13 @@ public final class TextSystemStyler<Interface: TextSystemInterface> {
 	///
 	/// You should invoke this method when the visible text in your system changes.
 	public func visibleContentDidChange() {
-		validator.workingRangeChanged()
+		let priorityRange = textSystem.visibleRange
+
+		validator.validate(.range(priorityRange), prioritizing: priorityRange)
 	}
 
+
 	public func invalidate(_ target: RangeTarget) {
 		validator.invalidate(target)
 	}
-
-	public var validationHandler: (NSRange) -> Void {
-		get { validator.validationHandler }
-		set { validator.validationHandler = newValue }
-	}
 }

Sources/Neon/ThreePhaseTextSystemStyler.swift

@@ -0,0 +1,57 @@
+import Foundation
+
+import RangeState
+
+@MainActor
+public final class ThreePhaseTextSystemStyler<Interface: TextSystemInterface> {
+	public typealias FallbackTokenProvider = (NSRange) -> TokenApplication
+	public typealias SecondaryValidationProvider = (NSRange) async -> TokenApplication
+
+	private let textSystem: Interface
+	private let validator: ThreePhaseRangeValidator<Interface.Content>
+
+	public init(
+		textSystem: Interface,
+		tokenProvider: TokenProvider,
+		fallbackHandler: @escaping FallbackTokenProvider,
+		secondaryValidationProvider: @escaping SecondaryValidationProvider
+	) {
+		self.textSystem = textSystem
+
+		let tokenValidator = TokenSystemValidator(
+			textSystem: textSystem,
+			tokenProvider: tokenProvider
+		)
+
+		self.validator = ThreePhaseRangeValidator(
+			configuration: .init(
+				versionedContent: textSystem.content,
+				provider: tokenValidator.validationProvider,
+				fallbackHandler: textSystem.validatorFallbackHandler(with: fallbackHandler),
+				secondaryProvider: textSystem.validatorSecondaryHandler(with: secondaryValidationProvider),
+				secondaryValidationDelay: 3.0,
+				priorityRangeProvider: { textSystem.visibleRange }
+			)
+		)
+	}
+
+	public func didChangeContent(in range: NSRange, delta: Int) {
+		validator.contentChanged(in: range, delta: delta)
+	}
+
+	public func invalidate(_ target: RangeTarget) {
+		validator.invalidate(target)
+	}
+
+	public func validate(_ target: RangeTarget) {
+		let priorityRange = textSystem.visibleRange
+
+		validator.validate(target, prioritizing: priorityRange)
+	}
+
+	public func validate() {
+		let priorityRange = textSystem.visibleRange
+
+		validator.validate(.range(priorityRange), prioritizing: priorityRange)
+	}
+}

Sources/Neon/Token.swift

@@ -44,10 +44,10 @@ extension TokenProvider {
 	public static var none: TokenProvider {
 		.init(
 			syncValue: { _ in
-				return TokenApplication(tokens: [])
+				return .noChange
 			},
 			asyncValue: { _, _ in
-				return TokenApplication(tokens: [])
+				return .noChange
 			}
 		)
 	}

Sources/Neon/TokenSystemValidator.swift

@@ -14,7 +14,7 @@ final class TokenSystemValidator<Interface: TextSystemInterface> {
 		self.tokenProvider = tokenProvider
 	}
 
-	var validationProvider: Validator.ValidationProvider {
+	var validationProvider: HybridValueProvider<Validator.ContentRange, Validation> {
 		.init(
 			syncValue: { self.validate($0) },
 			asyncValue: { range, _ in await self.validate(range)}
@@ -25,7 +25,7 @@ final class TokenSystemValidator<Interface: TextSystemInterface> {
 		textSystem.content.currentVersion
 	}
 
-	private func validate(_ range: Validator.ContentRange) -> Validator.Validation? {
+	private func validate(_ range: Validator.ContentRange) -> Validation? {
 		guard range.version == currentVersion else { return .stale }
 
 		guard let application = tokenProvider.sync(range.value) else {
@@ -37,7 +37,7 @@ final class TokenSystemValidator<Interface: TextSystemInterface> {
 		return .success(range.value)
 	}
 
-	private func validate(_ range: Validator.ContentRange) async -> Validator.Validation {
+	private func validate(_ range: Validator.ContentRange) async -> Validation {
 		guard range.version == currentVersion else { return .stale }
 
 		// https://github.com/apple/swift/pull/71143