Skip to content

Commit 867f46f

Browse files
noerwlafriks
noerw
authored and
lafriks
committed
Detect delimiter in CSV rendering (#7869)
* detect csv delimiter in csv rendering fixes #7868 * make linter happy * fix failing testcase & use ints where possible * expose markup type to template previously all markup had the .markdown class, which is incorrect, as it applies markdown CSS & JS logic to CSV rendering * fix build (missing `make css`) * ignore quoted csv content for delimiter scoring also fix html generation
1 parent 0a86d25 commit 867f46f

File tree

6 files changed

+76
-9
lines changed

6 files changed

+76
-9
lines changed

‎modules/markup/csv/csv.go‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,18 @@ import (
99
"encoding/csv"
1010
"html"
1111
"io"
12+
"regexp"
13+
"strings"
1214

1315
"code.gitea.io/gitea/modules/markup"
16+
"code.gitea.io/gitea/modules/util"
1417
)
1518

19+
var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
20+
1621
func init() {
1722
markup.RegisterParser(Parser{})
23+
1824
}
1925

2026
// Parser implements markup.Parser for orgmode
@@ -28,12 +34,13 @@ func (Parser) Name() string {
2834

2935
// Extensions implements markup.Parser
3036
func (Parser) Extensions() []string {
31-
return []string{".csv"}
37+
return []string{".csv", ".tsv"}
3238
}
3339

3440
// Render implements markup.Parser
35-
func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
41+
func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
3642
rd := csv.NewReader(bytes.NewReader(rawBytes))
43+
rd.Comma = p.bestDelimiter(rawBytes)
3744
var tmpBlock bytes.Buffer
3845
tmpBlock.WriteString(`<table class="table">`)
3946
for {
@@ -50,9 +57,57 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
5057
tmpBlock.WriteString(html.EscapeString(field))
5158
tmpBlock.WriteString("</td>")
5259
}
53-
tmpBlock.WriteString("<tr>")
60+
tmpBlock.WriteString("</tr>")
5461
}
5562
tmpBlock.WriteString("</table>")
5663

5764
return tmpBlock.Bytes()
5865
}
66+
67+
// bestDelimiter scores the input CSV data against delimiters, and returns the best match.
68+
// Reads at most 10k bytes & 10 lines.
69+
func (p Parser) bestDelimiter(data []byte) rune {
70+
maxLines := 10
71+
maxBytes := util.Min(len(data), 1e4)
72+
text := string(data[:maxBytes])
73+
text = quoteRegexp.ReplaceAllLiteralString(text, "")
74+
lines := strings.SplitN(text, "\n", maxLines+1)
75+
lines = lines[:util.Min(maxLines, len(lines))]
76+
77+
delimiters := []rune{',', ';', '\t', '|'}
78+
bestDelim := delimiters[0]
79+
bestScore := 0.0
80+
for _, delim := range delimiters {
81+
score := p.scoreDelimiter(lines, delim)
82+
if score > bestScore {
83+
bestScore = score
84+
bestDelim = delim
85+
}
86+
}
87+
88+
return bestDelim
89+
}
90+
91+
// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV
92+
func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) {
93+
countTotal := 0
94+
countLineMax := 0
95+
linesNotEqual := 0
96+
97+
for _, line := range lines {
98+
if len(line) == 0 {
99+
continue
100+
}
101+
102+
countLine := strings.Count(line, string(delim))
103+
countTotal += countLine
104+
if countLine != countLineMax {
105+
if countLineMax != 0 {
106+
linesNotEqual++
107+
}
108+
countLineMax = util.Max(countLine, countLineMax)
109+
}
110+
}
111+
112+
return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
113+
}

‎modules/markup/csv/csv_test.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,14 @@ import (
1313
func TestRenderCSV(t *testing.T) {
1414
var parser Parser
1515
var kases = map[string]string{
16-
"a": "<table class=\"table\"><tr><td>a</td><tr></table>",
17-
"1,2": "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
18-
"<br/>": "<table class=\"table\"><tr><td>&lt;br/&gt;</td><tr></table>",
16+
"a": "<table class=\"table\"><tr><td>a</td></tr></table>",
17+
"1,2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
18+
"1;2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
19+
"1\t2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
20+
"1|2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
21+
"1,2,3;4,5,6;7,8,9\na;b;c": "<table class=\"table\"><tr><td>1,2,3</td><td>4,5,6</td><td>7,8,9</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>",
22+
"\"1,2,3,4\";\"a\nb\"\nc;d": "<table class=\"table\"><tr><td>1,2,3,4</td><td>a\nb</td></tr><tr><td>c</td><td>d</td></tr></table>",
23+
"<br/>": "<table class=\"table\"><tr><td>&lt;br/&gt;</td></tr></table>",
1924
}
2025

2126
for k, v := range kases {

‎public/css/index.css‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,7 @@ footer .ui.left,footer .ui.right{line-height:40px}
489489
.repository.file.list .non-diff-file-content .view-raw img{padding:5px 5px 0 5px}
490490
.repository.file.list .non-diff-file-content .plain-text{padding:1em 2em 1em 2em}
491491
.repository.file.list .non-diff-file-content .plain-text pre{word-break:break-word;white-space:pre-wrap}
492+
.repository.file.list .non-diff-file-content .csv{overflow-x:auto}
492493
.repository.file.list .non-diff-file-content pre{overflow:auto}
493494
.repository.file.list .sidebar{padding-left:0}
494495
.repository.file.list .sidebar .octicon{width:16px}

‎public/less/_repository.less‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,10 @@
400400
}
401401
}
402402

403+
.csv {
404+
overflow-x: auto;
405+
}
406+
403407
pre {
404408
overflow: auto;
405409
}

‎routers/repo/view.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,9 @@ func renderDirectory(ctx *context.Context, treeLink string) {
162162
d, _ := ioutil.ReadAll(dataRc)
163163
buf = charset.ToUTF8WithFallback(append(buf, d...))
164164

165-
if markup.Type(readmeFile.Name()) != "" {
165+
if markupType := markup.Type(readmeFile.Name()); markupType != "" {
166166
ctx.Data["IsMarkup"] = true
167+
ctx.Data["MarkupType"] = string(markupType)
167168
ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas()))
168169
} else {
169170
ctx.Data["IsRenderedHTML"] = true
@@ -282,8 +283,9 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
282283

283284
readmeExist := markup.IsReadmeFile(blob.Name())
284285
ctx.Data["ReadmeExist"] = readmeExist
285-
if markup.Type(blob.Name()) != "" {
286+
if markupType := markup.Type(blob.Name()); markupType != "" {
286287
ctx.Data["IsMarkup"] = true
288+
ctx.Data["MarkupType"] = markupType
287289
ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeMetas()))
288290
} else if readmeExist {
289291
ctx.Data["IsRenderedHTML"] = true

‎templates/repo/view_file.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
</div>
4646
</h4>
4747
<div class="ui attached table unstackable segment">
48-
<div class="file-view {{if .IsMarkup}}markdown{{else if .IsRenderedHTML}}plain-text{{else if .IsTextFile}}code-view{{end}} has-emoji">
48+
<div class="file-view {{if .IsMarkup}}{{.MarkupType}}{{else if .IsRenderedHTML}}plain-text{{else if .IsTextFile}}code-view{{end}} has-emoji">
4949
{{if .IsMarkup}}
5050
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
5151
{{else if .IsRenderedHTML}}

0 commit comments

Comments
 (0)