Skip to content

Upstream project updates #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Jun 30, 2022
Merged

Upstream project updates #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
3502408
Fix StringIndexOutOfBoundsException in LineTokens
danipen Jun 30, 2022
90eaa40
Remove unused usings
danipen Jun 30, 2022
edae4dc
Statement unnecessarily nested within else clause
danipen Jun 30, 2022
2795017
Fix potential NRE
danipen Jun 30, 2022
0b79698
Rename types with 'I' prefix which are classes and not interfaces
danipen Jun 30, 2022
c48aaf2
Remove unused parameter
danipen Jun 30, 2022
dbe7a11
Simplify expression
danipen Jun 30, 2022
bdeecab
Remove unnecessary interface
danipen Jun 30, 2022
9c36e92
Refactor Raw object cloning
danipen Jun 30, 2022
c57dbe4
Improve TMModel/TokenizerThread thread-safety
danipen Jun 30, 2022
6ca2056
Add strikethrough font-style
danipen Jun 30, 2022
648ad95
Implement StackElementMetadata#toBinaryStr
danipen Jun 30, 2022
a95c97a
Apply upstream changes to ScopeListElement
danipen Jun 30, 2022
1243431
Improve regex parsing
danipen Jun 30, 2022
d6713e0
Remove unused setters
danipen Jun 30, 2022
7281717
Improve PListParser performance
danipen Jun 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 134 additions & 9 deletions src/TextMateSharp.Tests/Internal/Grammars/GrammarTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Collections.Generic;
using System.IO;
using TextMateSharp.Grammars;
using TextMateSharp.Internal.Grammars;
using TextMateSharp.Internal.Grammars.Reader;
using TextMateSharp.Internal.Themes.Reader;
using TextMateSharp.Internal.Types;
Expand All @@ -15,7 +16,104 @@ namespace TextMateSharp.Tests.Internal.Grammars
class GrammarTests
{
[Test]
public void ParseSimpleTokensTest()
public void StackElementMetadata_Test_Should_Work()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
}

[Test]
public void StackElementMetadata_Should_Allow_Overwrite_Language_Id()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);

value = StackElementMetadata.Set(value, 2, StandardTokenType.Other, FontStyle.NotSet, 0, 0);
AssertMetadataHasProperties(value, 2, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
}

[Test]
public void StackElementMetadata_Should_Allow_Overwrite_Token_Type()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);

value = StackElementMetadata.Set(value, 0, StandardTokenType.Comment, FontStyle.NotSet, 0, 0);
AssertMetadataHasProperties(value, 1, StandardTokenType.Comment, FontStyle.Underline | FontStyle.Bold, 101, 102);
}

[Test]
public void StackElementMetadata_Should_Allow_Overwrite_Font_Style()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);

value = StackElementMetadata.Set(value, 0, StandardTokenType.Other, FontStyle.None, 0, 0);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
}

[Test]
public void CanOverwriteFontStyleWithStrikethrough()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Strikethrough, 101, 102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Strikethrough, 101, 102);

value = StackElementMetadata.Set(value, 0, StandardTokenType.Other, FontStyle.None, 0, 0);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
}

[Test]
public void StackElementMetadata_Should_Allow_Overwrite_Foreground()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);

value = StackElementMetadata.Set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 5, 0);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 5, 102);
}

[Test]
public void StackElementMetadata_Should_Allow_Overwrite_Background()
{
int value = StackElementMetadata.Set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
102);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);

value = StackElementMetadata.Set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 0, 7);
AssertMetadataHasProperties(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 7);
}

[Test]
public void StackElementMetadata_Should_Work_At_Max_Values()
{
int maxLangId = 255;
int maxTokenType = StandardTokenType.Comment | StandardTokenType.Other | StandardTokenType.RegEx
| StandardTokenType.String;
int maxFontStyle = FontStyle.Bold | FontStyle.Italic | FontStyle.Underline;
int maxForeground = 511;
int maxBackground = 511;

int value = StackElementMetadata.Set(0, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
AssertMetadataHasProperties(value, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
}

[Test]
public void Convert_To_Binary_String_Should_Work()
{
string binValue1 = StackElementMetadata.ToBinaryStr(StackElementMetadata.Set(0, 0, 0, 0, 0, 511));
Assert.AreEqual("11111111100000000000000000000000", binValue1);

string binValue2 = StackElementMetadata.ToBinaryStr(StackElementMetadata.Set(0, 0, 0, 0, 511, 0));
Assert.AreEqual("00000000011111111100000000000000", binValue2);
}

[Test]
public void Parse_Simple_Tokens_Should_Work()
{
string line = "using System;";

Expand All @@ -41,7 +139,7 @@ public void ParseSimpleTokensTest()
}

[Test]
public void ParseCssTokensTest()
public void Parse_Css_Tokens_Should_Work()
{
string line =
"body { margin: 25px; }";
Expand All @@ -59,7 +157,7 @@ public void ParseCssTokensTest()
}

[Test]
public void TestMatchScopeName()
public void Match_Scope_Name_Should_Work()
{
Registry.Registry registry = new Registry.Registry(
new TestRegistry());
Expand All @@ -76,7 +174,7 @@ public void TestMatchScopeName()
}

[Test]
public void TestBatchGrammar()
public void Batch_Grammar_Should_Work()
{
string line =
"REM echo off";
Expand All @@ -92,7 +190,7 @@ public void TestBatchGrammar()
}

[Test]
public void ParseMultilineTokensTest()
public void Multiline_Tokens_Should_Work()
{
string[] lines = new string[]
{
Expand Down Expand Up @@ -143,12 +241,12 @@ public void ParseMultilineTokensTest()
}

[Test]
public void TokenizeUnicodeCommentsTest()
public void Tokenize_Unicode_Comments_Should_Work()
{
string text = "string s = \"chars: 安定させる\";";

Registry.Registry registry = new Registry.Registry(
new TestRegistry());
new TestRegistry());

IGrammar grammar = registry.LoadGrammar("source.cs");

Expand All @@ -164,7 +262,7 @@ public void TokenizeUnicodeCommentsTest()
}

[Test]
public void TokenizeUnicodeCommentsTest2()
public void Tokenize_Unicode_Comments_Should_Work2()
{
// TODO: fix parsing unicode characters
string text = "\"安安安\"";
Expand All @@ -179,7 +277,7 @@ public void TokenizeUnicodeCommentsTest2()
}

[Test]
public void GrammarInjectionTest()
public void Grammar_Should_Inject_Other_Grammars()
{
Registry.Registry registry = new Registry.Registry(
new TestRegistry());
Expand Down Expand Up @@ -278,6 +376,33 @@ static void AssertTokenValuesAreEqual(IToken token, int startIndex, int endIndex
}
}

static void AssertMetadataHasProperties(
int metadata,
int languageId,
int tokenType,
int fontStyle,
int foreground,
int background)
{
string actual = "{\n" +
"languageId: " + StackElementMetadata.GetLanguageId(metadata) + ",\n" +
"tokenType: " + StackElementMetadata.GetTokenType(metadata) + ",\n" +
"fontStyle: " + StackElementMetadata.GetFontStyle(metadata) + ",\n" +
"foreground: " + StackElementMetadata.GetForeground(metadata) + ",\n" +
"background: " + StackElementMetadata.GetBackground(metadata) + ",\n" +
"}";

string expected = "{\n" +
"languageId: " + languageId + ",\n" +
"tokenType: " + tokenType + ",\n" +
"fontStyle: " + fontStyle + ",\n" +
"foreground: " + foreground + ",\n" +
"background: " + background + ",\n" +
"}";

Assert.AreEqual(expected, actual, "equals for " + StackElementMetadata.ToBinaryStr(metadata));
}

class TestRegistry : IRegistryOptions
{
public IRawTheme GetTheme(string scopeName)
Expand Down
24 changes: 12 additions & 12 deletions src/TextMateSharp/Internal/Grammars/Grammar.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public class Grammar : IGrammar, IRuleFactoryHelper
private Dictionary<int?, Rule> _ruleId2desc;
private Dictionary<string, IRawGrammar> _includedGrammars;
private IGrammarRepository _grammarRepository;
private IRawGrammar _grammar;
private IRawGrammar _rawGrammar;
private List<Injection> _injections;
private ScopeMetadataProvider _scopeMetadataProvider;

Expand All @@ -30,7 +30,7 @@ public Grammar(IRawGrammar grammar, int initialLanguage, Dictionary<string, int>
_lastRuleId = 0;
_includedGrammars = new Dictionary<string, IRawGrammar>();
_grammarRepository = grammarRepository;
_grammar = InitGrammar(grammar, null);
_rawGrammar = InitGrammar(grammar, null);
_ruleId2desc = new Dictionary<int?, Rule>();
_injections = null;
}
Expand All @@ -53,21 +53,21 @@ public List<Injection> GetInjections()
{
this._injections = new List<Injection>();
// add injections from the current grammar
Dictionary<string, IRawRule> rawInjections = this._grammar.GetInjections();
Dictionary<string, IRawRule> rawInjections = this._rawGrammar.GetInjections();
if (rawInjections != null)
{
foreach (string expression in rawInjections.Keys)
{
IRawRule rule = rawInjections[expression];
CollectInjections(this._injections, expression, rule, this, this._grammar);
CollectInjections(this._injections, expression, rule, this, this._rawGrammar);
}
}

// add injection grammars contributed for the current scope
if (this._grammarRepository != null)
{
ICollection<string> injectionScopeNames = this._grammarRepository
.Injections(this._grammar.GetScopeName());
.Injections(this._rawGrammar.GetScopeName());
if (injectionScopeNames != null)
{
foreach (string injectionScopeName in injectionScopeNames)
Expand Down Expand Up @@ -149,7 +149,7 @@ public IRawGrammar GetExternalGrammar(string scopeName, IRawRepository repositor

private IRawGrammar InitGrammar(IRawGrammar grammar, IRawRule ruleBase)
{
grammar = Clone(grammar);
grammar = grammar.Clone();
if (grammar.GetRepository() == null)
{
((Raw)grammar).SetRepository(new Raw());
Expand Down Expand Up @@ -215,7 +215,7 @@ private object Tokenize(string lineText, StackElement prevState, bool emitBinary
if (rootScopeName == null)
return null;
ScopeMetadata rawRootMetadata = this._scopeMetadataProvider.GetMetadataForScope(rootScopeName);
int rootMetadata = ScopeListElement.mergeMetadata(defaultMetadata, null, rawRootMetadata);
int rootMetadata = ScopeListElement.MergeMetadata(defaultMetadata, null, rawRootMetadata);

ScopeListElement scopeList = new ScopeListElement(null, rootScopeName, rootMetadata);

Expand Down Expand Up @@ -249,8 +249,8 @@ private void GenerateRootId()
_isCompiling = true;
try
{
this._rootId = RuleFactory.GetCompiledRuleId(this._grammar.GetRepository().GetSelf(), this,
this._grammar.GetRepository());
this._rootId = RuleFactory.GetCompiledRuleId(this._rawGrammar.GetRepository().GetSelf(), this,
this._rawGrammar.GetRepository());
}
finally
{
Expand All @@ -260,17 +260,17 @@ private void GenerateRootId()

public string GetName()
{
return _grammar.GetName();
return _rawGrammar.GetName();
}

public string GetScopeName()
{
return _grammar.GetScopeName();
return _rawGrammar.GetScopeName();
}

public ICollection<string> GetFileTypes()
{
return _grammar.GetFileTypes();
return _rawGrammar.GetFileTypes();
}
}
}
17 changes: 8 additions & 9 deletions src/TextMateSharp/Internal/Grammars/LineTokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,19 +97,18 @@ private void ScanNext()
IOnigCaptureIndex[] captureIndices = r.CaptureIndexes;
int? matchedRuleId = r.MatchedRuleId;

bool hasAdvanced = (captureIndices != null && captureIndices.Length > 0)
? (captureIndices[0].End > _linePos)
: false;
bool hasAdvanced = captureIndices != null && captureIndices.Length > 0
&& captureIndices[0].End > _linePos;

if (matchedRuleId == -1)
{
// We matched the `end` for this rule => pop it
BeginEndRule poppedRule = (BeginEndRule)_stack.GetRule(_grammar);

/*
* if (logger.isEnabled()) { logger.log(" popping " + poppedRule.debugName +
* " - " + poppedRule.debugEndRegExp); }
*/
* if (logger.isEnabled()) { logger.log(" popping " + poppedRule.debugName +
* " - " + poppedRule.debugEndRegExp); }
*/

_lineTokens.Produce(_stack, captureIndices[0].Start);
_stack = _stack.setContentNameScopesList(_stack.NameScopesList);
Expand Down Expand Up @@ -251,7 +250,7 @@ private IMatchResult MatchRule(Grammar grammar, string lineText, in bool isFirst
if (rule == null)
return null;

ICompiledRule ruleScanner = rule.Compile(grammar, stack.EndRule, isFirstLine, linePos == anchorPosition);
CompiledRule ruleScanner = rule.Compile(grammar, stack.EndRule, isFirstLine, linePos == anchorPosition);

if (ruleScanner == null)
return null;
Expand Down Expand Up @@ -328,7 +327,7 @@ private IMatchInjectionsResult MatchInjections(List<Injection> injections, Gramm
continue;
}

ICompiledRule ruleScanner = grammar.GetRule(injection.RuleId).Compile(grammar, null, isFirstLine,
CompiledRule ruleScanner = grammar.GetRule(injection.RuleId).Compile(grammar, null, isFirstLine,
linePos == anchorPosition);
IOnigNextMatchResult matchResult = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos);

Expand Down Expand Up @@ -487,7 +486,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText,
for (int i = whileRules.Count - 1; i >= 0; i--)
{
WhileStack whileRule = whileRules[i];
ICompiledRule ruleScanner = whileRule.Rule.CompileWhile(grammar, whileRule.Stack.EndRule, isFirstLine,
CompiledRule ruleScanner = whileRule.Rule.CompileWhile(whileRule.Stack.EndRule, isFirstLine,
currentanchorPosition == linePos);
IOnigNextMatchResult r = ruleScanner.Scanner.FindNextMatchSync(lineText, linePos);

Expand Down
5 changes: 1 addition & 4 deletions src/TextMateSharp/Internal/Grammars/LineTokens.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
using System;
using System.Collections.Generic;

using TextMateSharp.Grammars;
using TextMateSharp.Internal.Utils;

namespace TextMateSharp.Internal.Grammars
{
Expand Down Expand Up @@ -68,7 +65,7 @@ public void ProduceFromScopes(ScopeListElement scopesList, int endIndex)

List<string> scopes = scopesList.GenerateScopes();

this._tokens.Add(new Token(this._lastTokenEndIndex, endIndex, scopes));
this._tokens.Add(new Token(this._lastTokenEndIndex >= 0 ? this._lastTokenEndIndex : 0, endIndex, scopes));
this._lastTokenEndIndex = endIndex;
}

Expand Down
Loading