Skip to content
This repository was archived by the owner on Jul 31, 2025. It is now read-only.

Commit c7514f9

Browse files
xibzxibz
committed
add support for utf8 characters
1 parent 0d4577f commit c7514f9

File tree

10 files changed

+398
-32
lines changed

10 files changed

+398
-32
lines changed

‎aws/session/internal/ini/ini_lexer_test.go

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package ini
2+
3+
import (
4+
"bytes"
5+
"io"
6+
"reflect"
7+
"testing"
8+
)
9+
10+
func TestTokenize(t *testing.T) {
11+
cases := []struct {
12+
r io.Reader
13+
expectedTokens []Token
14+
expectedError bool
15+
}{
16+
{
17+
r: bytes.NewBuffer([]byte(`x = 123`)),
18+
expectedTokens: []Token{
19+
literalToken{
20+
Value: UnionValue{
21+
Type: StringType,
22+
str: "x",
23+
},
24+
raw: "x",
25+
},
26+
wsToken{
27+
raw: " ",
28+
},
29+
opToken{
30+
opType: opTypeEqual,
31+
value: "=",
32+
},
33+
wsToken{
34+
raw: " ",
35+
},
36+
literalToken{
37+
Value: UnionValue{
38+
Type: IntegerType,
39+
integer: 123,
40+
},
41+
raw: "123",
42+
},
43+
},
44+
},
45+
{
46+
r: bytes.NewBuffer([]byte(`[ foo ]`)),
47+
expectedTokens: []Token{
48+
sepToken{
49+
sepType: sepTypeOpenBrace,
50+
value: "[",
51+
},
52+
wsToken{
53+
raw: " ",
54+
},
55+
literalToken{
56+
Value: UnionValue{
57+
Type: StringType,
58+
str: "foo",
59+
},
60+
raw: "foo",
61+
},
62+
wsToken{
63+
raw: " ",
64+
},
65+
sepToken{
66+
sepType: sepTypeCloseBrace,
67+
value: "]",
68+
},
69+
},
70+
},
71+
}
72+
73+
for _, c := range cases {
74+
lex := iniLexer{}
75+
tokens, err := lex.Tokenize(c.r)
76+
77+
if e, a := c.expectedError, err != nil; e != a {
78+
t.Errorf("expected %t, but received %t", e, a)
79+
}
80+
81+
if e, a := c.expectedTokens, tokens; !reflect.DeepEqual(e, a) {
82+
t.Errorf("expected %v, but received %v", e, a)
83+
}
84+
}
85+
}

‎aws/session/internal/ini/literal_tokens.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"strconv"
66
"strings"
77
"unicode/utf8"
8+
9+
"github.com/aws/aws-sdk-go/aws/awserr"
810
)
911

1012
var literalValues = []string{
@@ -47,12 +49,10 @@ func isNumberValue(b []byte) bool {
4749
helper.Determine(b[i])
4850
continue
4951
case 'e', 'E':
50-
if helper.Exists() {
52+
if err := helper.Determine(b[i]); err != nil {
5153
return false
5254
}
53-
5455
negativeIndex = 0
55-
helper.Determine(b[i])
5656
continue
5757
case 'b':
5858
if helper.hex {
@@ -66,10 +66,9 @@ func isNumberValue(b []byte) bool {
6666

6767
fallthrough
6868
case '.':
69-
if helper.Exists() {
69+
if err := helper.Determine(b[i]); err != nil {
7070
return false
7171
}
72-
helper.Determine(b[i])
7372
continue
7473
}
7574

@@ -86,8 +85,16 @@ func isNumberValue(b []byte) bool {
8685
return true
8786
}
8887

89-
func isValid(b byte) bool {
90-
return utf8.ValidRune(rune(b)) && b != '=' && b != '[' && b != ']' && b != ' ' && b != '\n'
88+
func isValid(b []byte) (bool, int, error) {
89+
r, size := utf8.DecodeRune(b)
90+
if !utf8.ValidRune(r) {
91+
return false, 0, awserr.New(ErrCodeParseError, "invalid character", nil)
92+
}
93+
return isValidRune(r), size, nil
94+
}
95+
96+
func isValidRune(r rune) bool {
97+
return r != '=' && r != '[' && r != ']' && r != ' ' && r != '\n'
9198
}
9299

93100
// UnionValueType is an enum that will signify what type
@@ -260,6 +267,8 @@ func (token literalToken) String() string {
260267
return fmt.Sprintf("%d", token.IntValue())
261268
case StringType:
262269
return fmt.Sprintf("%s", token.StringValue())
270+
case QuotedStringType:
271+
return fmt.Sprintf("%q", token.StringValue())
263272
case BoolType:
264273
return fmt.Sprintf("%t", token.BoolValue())
265274
}

‎aws/session/internal/ini/number_helper.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ func (b numberHelper) IsNegative() bool {
2929
}
3030

3131
func (b *numberHelper) Determine(c byte) error {
32+
if b.Exists() {
33+
return awserr.New(ErrCodeParseError, fmt.Sprintf("multiple number formats: 0%v", string(c)), nil)
34+
}
35+
3236
switch c {
3337
case 'b':
3438
b.binary = true

‎aws/session/internal/ini/number_helper_test.go

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package ini
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestNumberHelper(t *testing.T) {
8+
cases := []struct {
9+
b []byte
10+
determineIndex int
11+
12+
expectedExists []bool
13+
expectedErrors []bool
14+
expectedCorrects []bool
15+
expectedNegative bool
16+
expectedBase int
17+
}{
18+
{
19+
b: []byte("-10"),
20+
determineIndex: 0,
21+
expectedExists: []bool{
22+
false,
23+
false,
24+
false,
25+
},
26+
expectedErrors: []bool{
27+
false,
28+
false,
29+
false,
30+
},
31+
expectedCorrects: []bool{
32+
true,
33+
true,
34+
true,
35+
},
36+
expectedNegative: true,
37+
expectedBase: 10,
38+
},
39+
{
40+
b: []byte("0x10"),
41+
determineIndex: 1,
42+
expectedExists: []bool{
43+
false,
44+
false,
45+
true,
46+
true,
47+
},
48+
expectedErrors: []bool{
49+
false,
50+
false,
51+
false,
52+
false,
53+
},
54+
expectedCorrects: []bool{
55+
true,
56+
true,
57+
true,
58+
true,
59+
},
60+
expectedNegative: false,
61+
expectedBase: 16,
62+
},
63+
{
64+
b: []byte("0b101"),
65+
determineIndex: 1,
66+
expectedExists: []bool{
67+
false,
68+
false,
69+
true,
70+
true,
71+
true,
72+
},
73+
expectedErrors: []bool{
74+
false,
75+
false,
76+
false,
77+
false,
78+
false,
79+
},
80+
expectedCorrects: []bool{
81+
true,
82+
true,
83+
true,
84+
true,
85+
true,
86+
},
87+
expectedNegative: false,
88+
expectedBase: 2,
89+
},
90+
{
91+
b: []byte("0o271"),
92+
determineIndex: 1,
93+
expectedExists: []bool{
94+
false,
95+
false,
96+
true,
97+
true,
98+
true,
99+
},
100+
expectedErrors: []bool{
101+
false,
102+
false,
103+
false,
104+
false,
105+
false,
106+
},
107+
expectedCorrects: []bool{
108+
true,
109+
true,
110+
true,
111+
true,
112+
true,
113+
},
114+
expectedNegative: false,
115+
expectedBase: 8,
116+
},
117+
{
118+
b: []byte("99"),
119+
determineIndex: -1,
120+
expectedExists: []bool{
121+
false,
122+
false,
123+
},
124+
expectedErrors: []bool{
125+
false,
126+
false,
127+
},
128+
expectedCorrects: []bool{
129+
true,
130+
true,
131+
},
132+
expectedNegative: false,
133+
expectedBase: 10,
134+
},
135+
{
136+
b: []byte("0o2o71"),
137+
determineIndex: 1,
138+
expectedExists: []bool{
139+
false,
140+
false,
141+
true,
142+
true,
143+
true,
144+
true,
145+
},
146+
expectedErrors: []bool{
147+
false,
148+
false,
149+
false,
150+
false,
151+
false,
152+
true,
153+
},
154+
expectedCorrects: []bool{
155+
true,
156+
true,
157+
true,
158+
false,
159+
true,
160+
true,
161+
},
162+
expectedNegative: false,
163+
expectedBase: 8,
164+
},
165+
}
166+
167+
for _, c := range cases {
168+
helper := numberHelper{}
169+
170+
for i := 0; i < len(c.b); i++ {
171+
if e, a := c.expectedExists[i], helper.Exists(); e != a {
172+
t.Errorf("expected %t, but received %t", e, a)
173+
}
174+
175+
if i == c.determineIndex {
176+
if e, a := c.expectedErrors[i], helper.Determine(c.b[i]) != nil; e != a {
177+
t.Errorf("expected %t, but received %t", e, a)
178+
}
179+
} else {
180+
if e, a := c.expectedCorrects[i], helper.CorrectByte(c.b[i]); e != a {
181+
t.Errorf("expected %t, but received %t", e, a)
182+
}
183+
}
184+
}
185+
186+
if e, a := c.expectedNegative, helper.IsNegative(); e != a {
187+
t.Errorf("expected %t, but received %t", e, a)
188+
}
189+
190+
if e, a := c.expectedBase, helper.Base(); e != a {
191+
t.Errorf("expected %d, but received %d", e, a)
192+
}
193+
}
194+
}

‎aws/session/internal/ini/parse_stack.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ func (s *ParseStack) Push(ast AST) {
2525
s.container = append(s.container, ast)
2626
}
2727

28-
// Epsilon will push Start{} back to the stack and append the
29-
// AST to the list of completed statements
28+
// Epsilon will append the AST to the list of completed statements
3029
func (s *ParseStack) Epsilon(ast AST) {
3130
s.list = append(s.list, ast)
3231
}

0 commit comments

Comments
 (0)