Skip to content

Parse grammar definitions with System.Text.Json #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 25, 2023
Merged

Parse grammar definitions with System.Text.Json #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
646b6dc
Parse grammar definitions with System.Text.Json
Numpsy Oct 23, 2022
25fced0
Draft for loading the JSONPListParser using Sytem.Text.Json
danipen Mar 15, 2023
213b0fb
Try changing JSONPListParser to read the input data in chunks
Numpsy Mar 24, 2023
127c712
Remove unneeded stuff for AOT
danipen Mar 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
<PropertyGroup>
<LangVersion>latest</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<NewtonsoftJsonVersion>13.0.2</NewtonsoftJsonVersion>
<SystemTextJsonVersion>7.0.2</SystemTextJsonVersion>
</PropertyGroup>
</Project>
55 changes: 27 additions & 28 deletions src/TextMateSharp.Grammars/GrammarDefinition.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,29 @@
using System.Collections.Generic;

using Newtonsoft.Json;
using System.Text.Json.Serialization;

namespace TextMateSharp.Grammars
{
public class Engines
{
[JsonProperty("engines")]
[JsonPropertyName("engines")]
public string VsCode { get; set; }
}

public class Scripts
{
[JsonProperty("update-grammar")]
[JsonPropertyName("update-grammar")]
public string UpdateGrammar { get; set; }
}

public class Language
{
[JsonProperty("id")]
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonProperty("extensions")]
[JsonPropertyName("extensions")]
public List<string> Extensions { get; set; }
[JsonProperty("aliases")]
[JsonPropertyName("aliases")]
public List<string> Aliases { get; set; }
[JsonProperty("configuration")]
[JsonPropertyName("configuration")]
public string Configuration { get; set; }

public override string ToString()
Expand All @@ -38,61 +37,61 @@ public override string ToString()

public class Grammar
{
[JsonProperty("language")]
[JsonPropertyName("language")]
public string Language { get; set; }
[JsonProperty("scopeName")]
[JsonPropertyName("scopeName")]
public string ScopeName { get; set; }
[JsonProperty("path")]
[JsonPropertyName("path")]
public string Path { get; set; }
}

public class Snippet
{
[JsonProperty("language")]
[JsonPropertyName("language")]
public string Language { get; set; }
[JsonProperty("path")]
[JsonPropertyName("path")]
public string Path { get; set; }
}

public class Contributes
{
[JsonProperty("languages")]
[JsonPropertyName("languages")]
public List<Language> Languages { get; set; }
[JsonProperty("grammars")]
[JsonPropertyName("grammars")]
public List<Grammar> Grammars { get; set; }
[JsonProperty("snippets")]
[JsonPropertyName("snippets")]
public List<Snippet> Snippets { get; set; }
}

public class Repository
{
[JsonProperty("type")]
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonProperty("url")]
[JsonPropertyName("url")]
public string Url { get; set; }
}

public class GrammarDefinition
{
[JsonProperty("name")]
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonProperty("displayName")]
[JsonPropertyName("displayName")]
public string DisplayName { get; set; }
[JsonProperty("description")]
[JsonPropertyName("description")]
public string Description { get; set; }
[JsonProperty("version")]
[JsonPropertyName("version")]
public string Version { get; set; }
[JsonProperty("publisher")]
[JsonPropertyName("publisher")]
public string Publisher { get; set; }
[JsonProperty("license")]
[JsonPropertyName("license")]
public string License { get; set; }
[JsonProperty("engines")]
[JsonPropertyName("engines")]
public Engines Engines { get; set; }
[JsonProperty("scripts")]
[JsonPropertyName("scripts")]
public Scripts Scripts { get; set; }
[JsonProperty("contributes")]
[JsonPropertyName("contributes")]
public Contributes Contributes { get; set; }
[JsonProperty("repository")]
[JsonPropertyName("repository")]
public Repository Repository { get; set; }
}
}
35 changes: 10 additions & 25 deletions src/TextMateSharp.Grammars/RegistryOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;

using System.Text.Json;
using System.Text.Json.Serialization;
using TextMateSharp.Grammars.Resources;

using Newtonsoft.Json;

using TextMateSharp.Internal.Grammars.Reader;
using TextMateSharp.Internal.Themes.Reader;
using TextMateSharp.Internal.Types;
Expand Down Expand Up @@ -159,37 +158,16 @@ public IRawTheme GetDefaultTheme()

void InitializeAvailableGrammars()
{
KeepType<List<object>>();
KeepType<GrammarDefinition>();
KeepType<Repository>();
KeepType<Contributes>();
KeepType<Snippet>();
KeepType<Grammar>();
KeepType<Language>();
KeepType<Scripts>();
KeepType<Engines>();

var serializer = new JsonSerializer();

foreach (string grammar in GrammarNames.SupportedGrammars)
{
using (Stream stream = ResourceLoader.OpenGrammarPackage(grammar))
using (StreamReader reader = new StreamReader(stream))
using (JsonTextReader jsonTextReader = new JsonTextReader(reader))
{
GrammarDefinition definition = serializer.Deserialize<GrammarDefinition>(jsonTextReader);
GrammarDefinition definition = JsonSerializer.Deserialize<GrammarDefinition>(stream, GrammarDefinitionSerializationContext.Default.GrammarDefinition);
_availableGrammars.Add(grammar, definition);
}
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
void KeepType<
[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] T>()
{
// No-op
}

string GetGrammarFile(string scopeName)
{
foreach (string grammarName in _availableGrammars.Keys)
Expand Down Expand Up @@ -265,4 +243,11 @@ static bool HasGrammar(string id, List<Grammar> grammars)
return false;
}
}

// Enable Source Generator support for GrammarDefinition
[JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)]
[JsonSerializable(typeof(GrammarDefinition))]
internal sealed partial class GrammarDefinitionSerializationContext : JsonSerializerContext
{
}
}
3 changes: 2 additions & 1 deletion src/TextMateSharp.Grammars/TextMateSharp.Grammars.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@
<Authors>Daniel Peñalba</Authors>
<Company />
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<EnableTrimAnalyzer>true</EnableTrimAnalyzer>
</PropertyGroup>

<ItemGroup>
<EmbeddedResource Include="Resources\Themes\*.json" />
<EmbeddedResource Include="Resources\Grammars\**\*.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
<PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TextMateSharp\TextMateSharp.csproj" />
Expand Down
78 changes: 58 additions & 20 deletions src/TextMateSharp/Internal/Parser/Json/JSONPListParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
using System;
using System.IO;

using Newtonsoft.Json;
using System.Text.Json;

namespace TextMateSharp.Internal.Parser.Json
{
public class JSONPListParser<T>
{

private bool theme;
private readonly bool theme;

public JSONPListParser(bool theme)
{
Expand All @@ -18,46 +17,85 @@ public T Parse(StreamReader contents)
{
PList<T> pList = new PList<T>(theme);

JsonReader reader = new JsonTextReader(contents);
var buffer = new byte[2048];

JsonReaderOptions options = new JsonReaderOptions()
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip
};

int size = contents.BaseStream.Read(buffer, 0, buffer.Length);
var reader = new Utf8JsonReader(buffer.AsSpan(0, size), isFinalBlock: false, state: new JsonReaderState(options));

while (true)
{
if (!reader.Read())
int bytesRead = -1;

while (!reader.Read())
{
bytesRead = GetMoreBytesFromStream(contents.BaseStream, ref buffer, ref reader);

if (bytesRead == 0)
break;
}

if (bytesRead == 0)
break;

JsonToken nextToken = reader.TokenType;
var nextToken = reader.TokenType;
switch (nextToken)
{
case JsonToken.StartArray:
case JsonTokenType.StartArray:
pList.StartElement("array");
break;
case JsonToken.EndArray:
case JsonTokenType.EndArray:
pList.EndElement("array");
break;
case JsonToken.StartObject:
case JsonTokenType.StartObject:
pList.StartElement("dict");
break;
case JsonToken.EndObject:
case JsonTokenType.EndObject:
pList.EndElement("dict");
break;
case JsonToken.PropertyName:
case JsonTokenType.PropertyName:
pList.StartElement("key");
pList.AddString((string)reader.Value);
pList.AddString(reader.GetString());
pList.EndElement("key");
break;
case JsonToken.String:
case JsonTokenType.String:
pList.StartElement("string");
pList.AddString((string)reader.Value);
pList.AddString(reader.GetString());
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be possible to use CopyString instead of GetString here as well (pList.AddString appends the data to a StringBuilder, and it might be possible to use the array version of Append instead of the string version to reduce the number of temporary strings it creates)

pList.EndElement("string");
break;
case JsonToken.Null:
case JsonToken.Boolean:
case JsonToken.Integer:
case JsonToken.Float:
break;
}
}
return pList.GetResult();
}

private static int GetMoreBytesFromStream(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader)
{
int bytesRead;
if (reader.BytesConsumed < buffer.Length)
{
ReadOnlySpan<byte> leftover = buffer.AsSpan((int)reader.BytesConsumed);

if (leftover.Length == buffer.Length)
{
Array.Resize(ref buffer, buffer.Length * 2);
}

leftover.CopyTo(buffer);
bytesRead = stream.Read(buffer, leftover.Length, buffer.Length - leftover.Length);
reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead + leftover.Length), isFinalBlock: bytesRead == 0, reader.CurrentState);
}
else
{
bytesRead = stream.Read(buffer, 0, buffer.Length);
reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: bytesRead == 0, reader.CurrentState);
}

return bytesRead;
}
}
}
2 changes: 1 addition & 1 deletion src/TextMateSharp/TextMateSharp.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ TextMateSharp uses a wrapper around Oniguruma regex engine. Read below to learn
</ItemGroup>

<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
<PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
</ItemGroup>
<Import Project="..\..\build\SourceLink.props" />
<Import Project="..\..\build\Directory.Build.props" />
Expand Down