Adding font info in colorTokenCustomization

src/vs/editor/browser/services/abstractCodeEditorService.ts

@@ -163,7 +163,7 @@ export abstract class AbstractCodeEditorService extends Disposable implements IC
 				options: options || Object.create(null)
 			};
 			if (!parentTypeKey) {
-				provider = new DecorationTypeOptionsProvider(description, this._themeService, styleSheet, providerArgs);
+				provider = new DecorationTypeOptionsProvider(description, key, this._themeService, styleSheet, providerArgs);
 			} else {
 				provider = new DecorationSubTypeOptionsProvider(this._themeService, styleSheet, providerArgs);
 			}
@@ -178,6 +178,10 @@ export abstract class AbstractCodeEditorService extends Disposable implements IC
 		};
 	}
 
+	public hasDecorationType(key: string): boolean {
+		return this._decorationOptionProviders.has(key);
+	}
+
 	public listDecorationTypes(): string[] {
 		return Array.from(this._decorationOptionProviders.keys());
 	}
@@ -452,6 +456,7 @@ class DecorationTypeOptionsProvider implements IModelDecorationOptionsProvider {
 	private readonly _styleSheet: GlobalStyleSheet | RefCountedStyleSheet;
 	public refCount: number;
 
+	public typeKey: string;
 	public description: string;
 	public className: string | undefined;
 	public inlineClassName: string | undefined;
@@ -470,9 +475,9 @@ class DecorationTypeOptionsProvider implements IModelDecorationOptionsProvider {
 	public beforeInjectedText: InjectedTextOptions | undefined;
 	public afterInjectedText: InjectedTextOptions | undefined;
 
-	constructor(description: string, themeService: IThemeService, styleSheet: GlobalStyleSheet | RefCountedStyleSheet, providerArgs: ProviderArguments) {
+	constructor(description: string, decorationTypeKey: string, themeService: IThemeService, styleSheet: GlobalStyleSheet | RefCountedStyleSheet, providerArgs: ProviderArguments) {
 		this.description = description;
-
+		this.typeKey = decorationTypeKey;
 		this._styleSheet = styleSheet;
 		this._styleSheet.ref();
 		this.refCount = 0;
@@ -552,6 +557,7 @@ class DecorationTypeOptionsProvider implements IModelDecorationOptionsProvider {
 		}
 
 		return {
+			typeKey: this.typeKey,
 			description: this.description,
 			inlineClassName: this.inlineClassName,
 			beforeContentClassName: this.beforeContentClassName,

src/vs/editor/browser/services/codeEditorService.ts

@@ -45,6 +45,7 @@ export interface ICodeEditorService {
 
 	registerDecorationType(description: string, key: string, options: IDecorationRenderOptions, parentTypeKey?: string, editor?: ICodeEditor): void;
 	listDecorationTypes(): string[];
+	hasDecorationType(key: string): boolean;
 	removeDecorationType(key: string): void;
 	resolveDecorationOptions(typeKey: string, writable: boolean): IModelDecorationOptions;
 	resolveDecorationCSSRules(decorationTypeKey: string): CSSRuleList | null;

src/vs/editor/common/editorCommon.ts

@@ -732,7 +732,7 @@ export interface IDecorationInstanceRenderOptions extends IThemeDecorationInstan
 export interface IDecorationOptions {
 	range: IRange;
 	hoverMessage?: IMarkdownString | IMarkdownString[];
-	renderOptions?: IDecorationInstanceRenderOptions;
+	renderOptions?: IDecorationRenderOptions;
 }
 
 /**

src/vs/editor/common/languages.ts

@@ -63,6 +63,18 @@ export class TokenizationResult {
 	}
 }
 
+/**
+ * @internal
+ */
+export interface IVariableFontInfo {
+	readonly lineNumber?: number;
+	readonly startIndex: number;
+	readonly length: number;
+	readonly fontFamily: string | null;
+	readonly fontSize: string | null;
+	readonly lineHeight: number | null;
+}
+
 /**
  * @internal
  */
@@ -77,7 +89,8 @@ export class EncodedTokenizationResult {
 		 *
 		 */
 		public readonly tokens: Uint32Array,
-		public readonly endState: IState,
+		public readonly fontInfo: IVariableFontInfo[],
+		public readonly endState: IState
 	) {
 	}
 }
@@ -139,6 +152,8 @@ export interface IBackgroundTokenizer extends IDisposable {
 export interface IBackgroundTokenizationStore {
 	setTokens(tokens: ContiguousMultilineTokens[]): void;
 
+	setFontInfo(fontInfo: IVariableFontInfo[]): void;
+
 	setEndState(lineNumber: number, state: IState): void;
 
 	/**

src/vs/editor/common/languages/nullTokenize.ts

@@ -30,5 +30,5 @@ export function nullTokenizeEncoded(languageId: LanguageId, state: IState | null
 		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
 	) >>> 0;
 
-	return new EncodedTokenizationResult(tokens, state === null ? NullState : state);
+	return new EncodedTokenizationResult(tokens, [], state === null ? NullState : state);
 }

src/vs/editor/common/model.ts

@@ -16,7 +16,7 @@ import { Selection } from './core/selection.js';
 import { TextChange } from './core/textChange.js';
 import { WordCharacterClassifier } from './core/wordCharacterClassifier.js';
 import { IWordAtPosition } from './core/wordHelper.js';
-import { FormattingOptions } from './languages.js';
+import { FormattingOptions, IVariableFontInfo } from './languages.js';
 import { ILanguageSelection } from './languages/language.js';
 import { IBracketPairsTextModelPart } from './textModelBracketPairs.js';
 import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelFontChangedEvent, ModelInjectedTextChangedEvent, ModelLineHeightChangedEvent } from './textModelEvents.js';
@@ -145,6 +145,10 @@ export interface IModelDecorationMinimapOptions extends IDecorationOptions {
  * Options for a model decoration.
  */
 export interface IModelDecorationOptions {
+	/**
+	 * @internal
+	 */
+	typeKey?: string;
 	/**
 	 * A debug description that can be used for inspecting model decorations.
 	 * @internal
@@ -1317,6 +1321,14 @@ export interface ITextModel {
 	* @event
 	*/
 	readonly onDidChangeFont: Event<ModelFontChangedEvent>;
+	/**
+	* An event emitted when the font from decorations changes.
+	* This event is emitted only when adding, removing or changing a decoration
+	* and not when doing edits in the model (i.e. when decoration ranges change)
+	* @internal
+	* @event
+	*/
+	readonly onDidChangeTextMateFontInfo: Event<IVariableFontInfo[]>;
 	/**
 	 * An event emitted when the model options have changed.
 	 * @event

src/vs/editor/common/model/textModel.ts

@@ -24,7 +24,7 @@ import { Selection } from '../core/selection.js';
 import { TextChange } from '../core/textChange.js';
 import { EDITOR_MODEL_DEFAULTS } from '../core/misc/textModelDefaults.js';
 import { IWordAtPosition } from '../core/wordHelper.js';
-import { FormattingOptions } from '../languages.js';
+import { FormattingOptions, IVariableFontInfo } from '../languages.js';
 import { ILanguageSelection, ILanguageService } from '../languages/language.js';
 import { ILanguageConfigurationService } from '../languages/languageConfigurationRegistry.js';
 import * as model from '../model.js';
@@ -50,7 +50,18 @@ import { TokenArray } from '../tokens/lineTokens.js';
 import { SetWithKey } from '../../../base/common/collections.js';
 import { EditSources, TextModelEditSource } from '../textModelEditSource.js';
 import { TextEdit } from '../core/edits/textEdit.js';
+import { ICodeEditorService } from '../../browser/services/codeEditorService.js';
+import { IDecorationOptions, IDecorationRenderOptions } from '../editorCommon.js';
+import { IModelDeltaDecoration } from '../model.js';
+import { hash } from '../../../base/common/hash.js';
 
+/** TODO
+
+When you remove code and instead of cuntion write code that should not have a higher font size
+It remains at the higher font size <- need to fix this
+
+If can not use the code editor service, then how to define this code instead that uses it?
+ */
 export function createTextBufferFactory(text: string): model.ITextBufferFactory {
 	const builder = new PieceTreeTextBufferBuilder();
 	builder.acceptChunk(text);
@@ -238,6 +249,9 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	private readonly _onDidChangeFont: Emitter<ModelFontChangedEvent> = this._register(new Emitter<ModelFontChangedEvent>());
 	public get onDidChangeFont(): Event<ModelFontChangedEvent> { return this._onDidChangeFont.event; }
 
+	private readonly _onDidChangeTextMateFontInfo: Emitter<IVariableFontInfo[]> = this._register(new Emitter<IVariableFontInfo[]>());
+	public readonly onDidChangeTextMateFontInfo: Event<IVariableFontInfo[]> = this._onDidChangeTextMateFontInfo.event;
+
 	private readonly _eventEmitter: DidChangeContentEmitter = this._register(new DidChangeContentEmitter());
 	public onDidChangeContent(listener: (e: IModelContentChangedEvent) => void): IDisposable {
 		return this._eventEmitter.slowEvent((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
@@ -290,6 +304,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	private _decorations: { [decorationId: string]: IntervalNode };
 	private _decorationsTree: DecorationsTrees;
 	private readonly _decorationProvider: ColorizedBracketPairsDecorationProvider;
+	private readonly _decorationIdToTypeKey = new Map<string, string>();
+	private readonly _typeKeyToCount = new Map<string, number>();
 	//#endregion
 
 	private readonly _tokenizationTextModelPart: TokenizationTextModelPart;
@@ -308,6 +324,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		languageIdOrSelection: string | ILanguageSelection,
 		creationOptions: model.ITextModelCreationOptions,
 		associatedResource: URI | null = null,
+		@ICodeEditorService private readonly _codeEditorService: ICodeEditorService,
 		@IUndoRedoService private readonly _undoRedoService: IUndoRedoService,
 		@ILanguageService private readonly _languageService: ILanguageService,
 		@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
@@ -365,6 +382,48 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 			this._attachedViews
 		);
 
+		const decorationDescription = 'text-mate based syntactial font decorations';
+		const textMateFontDecorationsKey = 'text-mate-font-decorations';
+		this._codeEditorService.registerDecorationType(decorationDescription, textMateFontDecorationsKey, {});
+		this._register(this._tokenizationTextModelPart.onDidChangeFontInfo(fontChanges => {
+			const decorations: IDecorationOptions[] = [];
+			const linesChanges: Set<number> = new Set<number>();
+			for (const fontInfo of fontChanges) {
+				const lineNumber = fontInfo.lineNumber;
+				if (lineNumber === undefined) {
+					continue;
+				}
+				let lastOffset: number = 0;
+				if (lineNumber > 1) {
+					const lastPositionOnLine = new Position(lineNumber - 1, this.getLineMaxColumn(lineNumber - 1));
+					lastOffset = this.getOffsetAt(lastPositionOnLine);
+				}
+				const startIndex = lastOffset + fontInfo.startIndex + 1;
+				const endIndex = startIndex + fontInfo.length;
+				const startPosition = this.getPositionAt(startIndex);
+				const endPosition = this.getPositionAt(endIndex);
+				const range = Range.fromPositions(startPosition, endPosition);
+				const renderOptions: IDecorationRenderOptions = {
+					lineHeight: fontInfo.lineHeight ?? undefined,
+					fontSize: fontInfo.fontSize ?? undefined,
+					fontFamily: fontInfo.fontFamily ?? undefined,
+				};
+				decorations.push({ range, renderOptions });
+				linesChanges.add(lineNumber);
+			}
+			console.log('onDidChangeTextMateFontInfo e : ', fontChanges);
+			console.log('codeEditorWidget decorations : ', decorations);
+			/**
+			Essentially we just want to know what decoration to change, if it is one we already had, or if we should create a new one
+			Actually what I do is get decorations on the line that has changed, find those that touch the font info, remove them and add new decorations
+			You can find those that touch the font info by looking in a set which I store
+			Use the method `_getDecorationsInRange`
+
+			First place this code into the text model.
+			Then work on the decorations changing ranges and updating that
+			 */
+			this.setDecorationsByType(decorationDescription, textMateFontDecorationsKey, linesChanges, decorations);
+		}));
 		this._isTooLargeForSyncing = (bufferTextLength > TextModel._MODEL_SYNC_LIMIT);
 
 		this._versionId = 1;
@@ -578,6 +637,57 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		}
 	}
 
+	public setDecorationsByType(description: string, decorationTypeKey: string, linesChanges: Set<number>, decorationOptions: IDecorationOptions[]): void {
+		const newModelDecorations: IModelDeltaDecoration[] = [];
+		for (const decorationOption of decorationOptions) {
+			const decorationType = decorationTypeKey + '-' + hash(decorationOption.renderOptions).toString(16);
+			if (!this._codeEditorService.hasDecorationType(decorationType)) {
+				this._codeEditorService.registerDecorationType(description, decorationType, decorationOption.renderOptions!);
+			}
+			newModelDecorations.push({ range: decorationOption.range, options: this._codeEditorService.resolveDecorationOptions(decorationType, false) });
+		}
+		console.log('newModelDecorations : ', newModelDecorations);
+		const oldDecorationsIds: string[] = [];
+		for (const lineNumber of linesChanges) {
+			const decorationsOnLine = this.getDecorationsInRange(new Range(lineNumber, 1, lineNumber, this.getLineMaxColumn(lineNumber)));
+			console.log('decorationsOnLine : ', decorationsOnLine);
+			for (const decoration of decorationsOnLine) {
+				// Need to delete this old decoration
+				const decorationId = decoration.id;
+				if (this._decorationIdToTypeKey.has(decorationId)) {
+					oldDecorationsIds.push(decorationId);
+					this._decorationIdToTypeKey.delete(decorationId);
+					const typeKey = this._decorationIdToTypeKey.get(decorationId)!;
+					if (this._typeKeyToCount.has(typeKey)) {
+						const count = this._typeKeyToCount.get(typeKey)! - 1;
+						if (count === 0) {
+							this._codeEditorService.removeDecorationType(typeKey);
+							this._typeKeyToCount.delete(typeKey);
+						} else {
+							this._typeKeyToCount.set(typeKey, count);
+						}
+					}
+				}
+			}
+		}
+		console.log('oldDecorationsIds : ', oldDecorationsIds);
+		const newDecorationIds = this.changeDecorations(accessor => accessor.deltaDecorations(oldDecorationsIds, newModelDecorations));
+		if (newDecorationIds) {
+			for (const newDecorationId of newDecorationIds) {
+				const options = this.getDecorationOptions(newDecorationId);
+				if (!options) {
+					continue;
+				}
+				const typeKey = options.typeKey;
+				if (!typeKey) {
+					continue;
+				}
+				this._decorationIdToTypeKey.set(newDecorationId, typeKey);
+				this._typeKeyToCount.set(typeKey, (this._typeKeyToCount.get(typeKey) || 0) + 1);
+			}
+		}
+	}
+
 	public onBeforeAttached(): model.IAttachedView {
 		this._attachedEditorCount++;
 		if (this._attachedEditorCount === 1) {
@@ -2394,6 +2504,7 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
 	public static createDynamic(options: model.IModelDecorationOptions): ModelDecorationOptions {
 		return new ModelDecorationOptions(options);
 	}
+	readonly typeKey: string | undefined;
 	readonly description: string;
 	readonly blockClassName: string | null;
 	readonly blockIsAfterEnd: boolean | null;
@@ -2432,6 +2543,7 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
 	readonly textDirection?: model.TextDirection | null | undefined;
 
 	private constructor(options: model.IModelDecorationOptions) {
+		this.typeKey = options.typeKey;
 		this.description = options.description;
 		this.blockClassName = options.blockClassName ? cleanClassName(options.blockClassName) : null;
 		this.blockDoesNotCollapse = options.blockDoesNotCollapse ?? null;

src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts

@@ -9,7 +9,7 @@ import { Emitter, Event } from '../../../../base/common/event.js';
 import { Disposable } from '../../../../base/common/lifecycle.js';
 import { LineRange } from '../../core/ranges/lineRange.js';
 import { StandardTokenType } from '../../encodedTokenAttributes.js';
-import { ILanguageIdCodec } from '../../languages.js';
+import { ILanguageIdCodec, IVariableFontInfo } from '../../languages.js';
 import { IAttachedView } from '../../model.js';
 import { TextModel } from '../textModel.js';
 import { IModelContentChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js';
@@ -145,6 +145,10 @@ export abstract class AbstractSyntaxTokenBackend extends Disposable {
 	/** @internal, should not be exposed by the text model! */
 	public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event;
 
+	protected readonly _onDidChangeFontInfo: Emitter<IVariableFontInfo[]> = this._register(new Emitter<IVariableFontInfo[]>());
+	/** @internal, should not be exposed by the text model! */
+	public readonly onDidChangeFontInfo: Event<IVariableFontInfo[]> = this._onDidChangeFontInfo.event;
+
 	constructor(
 		protected readonly _languageIdCodec: ILanguageIdCodec,
 		protected readonly _textModel: TextModel,

src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts

@@ -27,6 +27,7 @@ import { IInstantiationService } from '../../../../platform/instantiation/common
 import { TokenizerSyntaxTokenBackend } from './tokenizerSyntaxTokenBackend.js';
 import { ITreeSitterLibraryService } from '../../services/treeSitter/treeSitterLibraryService.js';
 import { derived, IObservable, ISettableObservable, observableValue } from '../../../../base/common/observable.js';
+import { IVariableFontInfo } from '../../languages.js';
 
 export class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {
 	private readonly _semanticTokens: SparseTokensStore;
@@ -40,6 +41,9 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
 	private readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent>;
 	public readonly onDidChangeTokens: Event<IModelTokensChangedEvent>;
 
+	private readonly _onDidChangeFontInfo: Emitter<IVariableFontInfo[]> = this._register(new Emitter<IVariableFontInfo[]>());
+	public readonly onDidChangeFontInfo: Event<IVariableFontInfo[]> = this._onDidChangeFontInfo.event;
+
 	public readonly tokens: IObservable<AbstractSyntaxTokenBackend>;
 	private readonly _useTreeSitter: IObservable<boolean>;
 	private readonly _languageIdObs: ISettableObservable<string>;
@@ -80,6 +84,9 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
 			reader.store.add(tokens.onDidChangeTokens(e => {
 				this._emitModelTokensChangedEvent(e);
 			}));
+			reader.store.add(tokens.onDidChangeFontInfo(e => {
+				this._onDidChangeFontInfo.fire(e);
+			}));
 
 			reader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => {
 				this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
@@ -104,12 +111,15 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
 		this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event;
 		this._onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());
 		this.onDidChangeTokens = this._onDidChangeTokens.event;
+		this._onDidChangeFontInfo = this._register(new Emitter<IVariableFontInfo[]>());
+		this.onDidChangeFontInfo = this._onDidChangeFontInfo.event;
 	}
 
 	_hasListeners(): boolean {
 		return (this._onDidChangeLanguage.hasListeners()
 			|| this._onDidChangeLanguageConfiguration.hasListeners()
-			|| this._onDidChangeTokens.hasListeners());
+			|| this._onDidChangeTokens.hasListeners())
+			|| this._onDidChangeFontInfo.hasListeners();
 	}
 
 	public handleLanguageConfigurationServiceChange(e: LanguageConfigurationServiceChangeEvent): void {