Reduce heap allocs

.gitignore

@@ -380,3 +380,6 @@ onigwrap/src/onigwrap.lib
 onigwrap/src/onigwrap.exp
 onigwrap/src/onigwrap.dll
 onigwrap/src/onig_s.lib
+onigwrap/src/arm_libonigwrap.dylib
+onigwrap/src/x86_libonigwrap.dylib
+.idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml

onigwrap/README.md

@@ -25,17 +25,18 @@ autoreconf -vfi
 ./configure CC="gcc -arch x86_64 -arch arm64"
 make 
 ```
-Copy `$libs/libonig.a` to the onigwrap folder:
+Copy `$libs/libonig.a` and `src/oniguruma.h` to the onigwrap folder:
 ```
 cp src/.libs/libonig.a ../TextMateSharp/onigwrap/src
+cp src/oniguruma.h ../TextMateSharp/onigwrap/src
 ```
 
 Now we build onigwrap:
 
 2. Compile onigwrap in the following way:
 
 ```
-
+cd ../TextMateSharp/onigwrap/src
 clang -target x86_64-apple-macos10.12 -dynamiclib -L. -lonig -o x86_libonigwrap.dylib onigwrap.c
 clang -target arm64-apple-macos11 -dynamiclib -L. -lonig -o arm_libonigwrap.dylib onigwrap.c 
 lipo -create -output libonigwrap.dylib x86_libonigwrap.dylib arm_libonigwrap.dylib
@@ -60,13 +61,22 @@ Windows
 
 Then build and configure oniguruma [following the instructions](https://github.com/kkos/oniguruma#case-3-windows-6432bit-platform-visual-studio) on the Oniguruma repository.
 
-Copy `onig\_s.lib` and `oniguruma.h` to the `src` folder.
+Copy `onig_s.lib` and `oniguruma.h` to the `src` folder.
+```
+copy onig_s.lib ..\..\source\repos\TextMateSharp\onigwrap\src
+copy src\oniguruma.h ..\..\source\repos\TextMateSharp\onigwrap\src
+```
 
 Build onigwrap:
 
 `cl.exe /DONIG_EXTERN=extern /D_USRDLL /D_WINDLL onigwrap.c /link onig_s.lib /DLL /OUT:onigwrap.dll`
 
 Copy onigwrap.dll to the folder with your binary.
+```
+copy onigwrap.dll ..\..\src\TextMateSharp\Internal\Oniguruma\Native\win-x64
+```
+
+Repeat the same process for x86 platform:
 
 Linux
 -----
@@ -75,15 +85,26 @@ Build and configure oniguruma [following the instructions](https://github.com/kk
 
 We need to prepare onig for static linking though, so add `-fPIC` to the `CFLAGS`. If your Mono version is 32bit, make sure to add `-m32` to the `CFLAGS` too. (You may need to install a package like `gcc-multilib` to make the build work with `-m32`.)
 
-`./configure "CFLAGS=-fPIC"`
+```
+./configure "CFLAGS=-fPIC"
+```
 
-Copy .libs/libonig.a to the onigwrap folder.
+Copy .libs/libonig.a to the onigwrap folder:
+```
+cp src/.libs/libonig.a ../TextMateSharp/onigwrap/src
+cp src/oniguruma.h ../TextMateSharp/onigwrap/src
+```
 
 Build onigwrap:
 
-`gcc -shared -fPIC onigwrap.c libonig.a -o libonigwrap.so`
+```
+gcc -shared -fPIC onigwrap.c libonig.a -o libonigwrap.so
+```
 
-Copy `libonigwrap.so` alongside your binary.
+Copy `libonigwrap.so` alongside your binary:
+```
+cp libonigwrap.so ../../src/TextMateSharp/Internal/Oniguruma/Native/linux/
+```
 
 Web Assembly
 -----
@@ -95,6 +116,7 @@ In order to update web assembly native assets, you need to do the following:
 ```
 make distclean
 autoreconf -vfi
+source ../emsdk/emsdk_env.sh
 emconfigure ./configure
 emmake make
 ```
@@ -105,3 +127,5 @@ Then:
 
 ```
 cp src/.libs/libonig.a ../TextMateSharp/src/TextMateSharp.Wasm/
+cp src/oniguruma.h ../TextMateSharp/onigwrap/src
+```

onigwrap/src/oniguruma.h

@@ -1087,4 +1087,4 @@ int onig_setup_builtin_monitors_by_ascii_encoded_name P_((void* fp));
 }
 #endif
 
-#endif /* ONIGURUMA_H */
+#endif /* ONIGURUMA_H */

onigwrap/src/onigwrap.c

@@ -29,6 +29,11 @@ regex_t *onigwrap_create(char *pattern, int len, int ignoreCase, int multiline)
 	return reg;
 }
 
+OnigRegion* onigwrap_region_new()
+{
+	return onig_region_new();
+}
+
 void onigwrap_region_free(OnigRegion *region)	
 {
 	onig_region_free(region, 1);
@@ -39,43 +44,15 @@ void onigwrap_free(regex_t *reg)
 	onig_free(reg);
 }
 
-int onigwrap_index_in(regex_t *reg, char *charPtr, int offset, int length)
-{
-	OnigUChar *stringStart  = (OnigUChar*) charPtr;
-	OnigUChar *stringEnd    = (OnigUChar*) (charPtr + length);
-	OnigUChar *stringOffset = (OnigUChar*) (charPtr + offset);
-	OnigUChar *stringRange  = (OnigUChar*) stringEnd;
-
-	OnigRegion *region = onig_region_new();
-	int result = onig_search(
-        reg,
-        stringStart,
-        stringEnd,
-        stringOffset,
-        stringRange,
-        region,
-        ONIG_OPTION_NONE);
-
-	onig_region_free(region, 1);
-
-	if (result >= 0)
-		return result >> 1;
-	if (result == ONIG_MISMATCH)
-		return -1;
-	return -2;
-}
-
-OnigRegion *onigwrap_search(regex_t *reg, char *charPtr, int offset, int length)
+int onigwrap_search(regex_t *reg, char *charPtr, int offset, int length, OnigRegion *region)
 {
 	OnigUChar *stringStart  = (OnigUChar*) charPtr;
 	OnigUChar *stringEnd    = (OnigUChar*) (charPtr + length);
 	OnigUChar *stringOffset = (OnigUChar*) (charPtr + offset);
 	OnigUChar *stringRange  = (OnigUChar*) stringEnd;
 
-	OnigRegion *region = onig_region_new();
-
 	int result = onig_search(reg, stringStart, stringEnd, stringOffset, stringRange, region, ONIG_OPTION_NONE);
-	return region;
+	return result;
 }
 
 int onigwrap_num_regs(OnigRegion *region)
@@ -92,7 +69,7 @@ int onigwrap_pos(OnigRegion *region, int nth)
 			return result;
 		return result >> 1;
 	}
-	return -3;
+	return -1;
 }
 
 int onigwrap_len(OnigRegion *region, int nth)
@@ -102,5 +79,5 @@ int onigwrap_len(OnigRegion *region, int nth)
 		int result = region->end[nth] - region->beg[nth];
 		return result >> 1;
 	}
-	return -4;
+	return -2;
 }

onigwrap/src/onigwrap.h

@@ -10,16 +10,16 @@ ONIGWRAP_EXTERN
 regex_t *onigwrap_create(char *pattern, int len, int ignoreCase, int multiline);
 
 ONIGWRAP_EXTERN
-void onigwrap_region_free(OnigRegion *region);
+OnigRegion* onigwrap_region_new();
 
 ONIGWRAP_EXTERN
-void onigwrap_free(regex_t *reg);
+void onigwrap_region_free(OnigRegion *region);
 
 ONIGWRAP_EXTERN
-int onigwrap_index_in(regex_t *reg, char *charPtr, int offset, int length);
+void onigwrap_free(regex_t *reg);
 
 ONIGWRAP_EXTERN
-OnigRegion *onigwrap_search(regex_t *reg, char *charPtr, int offset, int length);
+int onigwrap_search(regex_t *reg, char *charPtr, int offset, int length, OnigRegion *region);
 
 ONIGWRAP_EXTERN
 int onigwrap_num_regs(OnigRegion *region);

src/TextMateSharp.Tests/Internal/Grammars/GrammarTests.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -18,7 +18,7 @@ namespace TextMateSharp.Tests.Internal.Grammars
     class GrammarTests
     {
         [Test]
-        public void Parse_Simple_Tokens_Should_Work()
+        public void Parse_Using_Statement_Should_Generate_Tokens()
         {
             string line = "using System;";
 
@@ -44,7 +44,7 @@ public void Parse_Simple_Tokens_Should_Work()
         }
 
         [Test]
-        public void Parse_Css_Tokens_Should_Work()
+        public void Parse_Css_Statement_Should_Generate_Tokens()
         {
             string line =
                 "body { margin: 25px; }";
@@ -62,13 +62,11 @@ public void Parse_Css_Tokens_Should_Work()
         }
 
         [Test]
-        public void Match_Scope_Name_Should_Work()
+        public void Match_Scope_Should_Generate_Empty_Color()
         {
             Registry.Registry registry = new Registry.Registry(
                 new TestRegistry());
 
-            IGrammar grammar = registry.LoadGrammar("source.css");
-
             Theme theme = registry.GetTheme();
             List<ThemeTrieElementRule> themeRules =
                 theme.Match(new string[] { "support.type.property-name.css" });
@@ -79,7 +77,7 @@ public void Match_Scope_Name_Should_Work()
         }
 
         [Test]
-        public void Batch_Grammar_Should_Work()
+        public void Parse_Batch_Statement_Should_Generate_Tokens()
         {
             string line =
                 "REM echo off";
@@ -95,7 +93,7 @@ public void Batch_Grammar_Should_Work()
         }
 
         [Test]
-        public void Multiline_Tokens_Should_Work()
+        public void Parse_Multiline_Text_Should_Generate_Tokens()
         {
             string[] lines = new string[]
             {
@@ -146,7 +144,7 @@ public void Multiline_Tokens_Should_Work()
         }
 
         [Test]
-        public void Tokenize_Unicode_Comments_Should_Work()
+        public void Parse_Unicode_String_Should_Generate_Tokens()
         {
             string text = "string s = \"chars: 安定させる\";";
 
@@ -167,22 +165,34 @@ public void Tokenize_Unicode_Comments_Should_Work()
         }
 
         [Test]
-        public void Tokenize_Unicode_Comments_Should_Work2()
+        public void Parse_Unicode_Comments_Should_Generate_Tokens()
         {
-            // TODO: fix parsing unicode characters
-            string text = "\"安安安\"";
-            //string text = "string s = \"chars: 12345\";";
+            string text = "//安安安";
 
             Registry.Registry registry = new Registry.Registry(
                             new TestRegistry());
 
             IGrammar grammar = registry.LoadGrammar("source.cs");
 
             ITokenizeLineResult lineTokens = grammar.TokenizeLine(text);
+
+            IToken[] tokens = lineTokens.Tokens;
+
+            Assert.AreEqual(2, tokens.Length);
+
+            AssertTokenValuesAreEqual(tokens[0],
+                0, 2,
+                "source.cs",
+                "comment.line.double-slash.cs",
+                "punctuation.definition.comment.cs");
+
+            AssertTokenValuesAreEqual(tokens[1], 
+                2, 5,
+                "source.cs", "comment.line.double-slash.cs");
         }
 
         [Test]
-        public void Grammar_Should_Inject_Other_Grammars()
+        public void Injected_Grammars_Should_Generate_Tokens()
         {
             Registry.Registry registry = new Registry.Registry(
                 new TestRegistry());

src/TextMateSharp.Tests/Internal/Grammars/Reader/GrammarReaderTests.cs

@@ -7,12 +7,12 @@
 using TextMateSharp.Internal.Types;
 using TextMateSharp.Tests.Resources;
 
-namespace TextMateSharp.Tests.Internal.Parser.Json
+namespace TextMateSharp.Tests.Internal.Grammars.Reader
 {
     class GrammarReaderTests
     {
         [Test]
-        public void TestReadCharpGrammar()
+        public void Read_Csharp_Grammar_Should_Not_Throw_Any_Exception()
         {
             using (Stream ms = ResourceReader.OpenStream("csharp.tmLanguage.json"))
             using (StreamReader reader = new StreamReader(ms))
@@ -23,7 +23,7 @@ public void TestReadCharpGrammar()
         }
 
         [Test]
-        public void TestReadSimpleGrammar()
+        public void Read_Csharp_Grammar_Should_Be_Well_Formed()
         {
             using (Stream ms = GenerateStreamFromString(_json))
             using (StreamReader reader = new StreamReader(ms))
@@ -112,7 +112,5 @@ static Stream GenerateStreamFromString(string s)
                       }
                 }
             }".Replace("'", "\"");
-
-        public int JSONPListParserIRawGrammar { get; private set; }
     }
 }