grindlemire/go-tui code browser

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

// Package highlight is a small, dependency-free syntax tokenizer for fenced code
// blocks. It does not import the tui package; the root package maps its tokens to
// colored spans. Output is "syntax coloring", not a full parser.
package highlight

import (
	"strings"
	"unicode"
)

// Kind is the small token taxonomy emitted by the lexers.
type Kind int

const (
	KindPlain    Kind = iota // whitespace, punctuation-free text, uncolored
	KindKeyword              // language keywords
	KindString               // string/char/template literals
	KindComment              // line and block comments
	KindNumber               // numeric literals
	KindLiteral              // true/false/null/nil, builtins, shell $VARs
	KindType                 // type and function/method names (heuristic)
	KindOperator             // operators and punctuation
	KindKey                  // JSON object keys
)

// Token is a contiguous run of source text classified as one Kind.
type Token struct {
	Kind Kind
	Text string
}

// Tokenize returns one []Token per line of code. Newlines are line separators
// and never appear in token Text. Lexer state carries across lines, so multi-line
// strings and comments tokenize correctly. Unknown or empty lang yields each line
// as a single KindPlain token. The concatenation of a line's token Text always
// equals the original line.
func Tokenize(lang, code string) [][]Token {
	lex := lexerFor(lang)
	if lex == nil {
		return plainLines(code)
	}
	return splitLines(lex(code))
}

func lexerFor(lang string) func(string) []Token {
	switch normalizeLang(lang) {
	case "go":
		return lexGo
	case "json":
		return lexJSON
	case "bash":
		return lexBash
	case "js":
		return lexJS
	}
	return nil
}

func normalizeLang(lang string) string {
	switch strings.ToLower(strings.TrimSpace(lang)) {
	case "go", "golang":
		return "go"
	case "json":
		return "json"
	case "bash", "sh", "shell", "zsh":
		return "bash"
	case "js", "javascript", "ts", "typescript", "jsx", "tsx":
		return "js"
	}
	return ""
}

// plainLines wraps each line of code as a single plain token (fallback path).
func plainLines(code string) [][]Token {
	lines := strings.Split(code, "\n")
	out := make([][]Token, len(lines))
	for i, ln := range lines {
		if ln == "" {
			out[i] = []Token{}
		} else {
			out[i] = []Token{{Kind: KindPlain, Text: ln}}
		}
	}
	return out
}

// splitLines turns a flat token stream (whose Text may contain newlines) into one
// []Token per line, splitting any newline-spanning token into same-kind pieces.
func splitLines(toks []Token) [][]Token {
	out := [][]Token{}
	cur := []Token{}
	for _, t := range toks {
		parts := strings.Split(t.Text, "\n")
		for k, p := range parts {
			if k > 0 {
				out = append(out, cur)
				cur = []Token{}
			}
			if p != "" {
				cur = append(cur, Token{Kind: t.Kind, Text: p})
			}
		}
	}
	out = append(out, cur)
	return out
}

// --- shared scanner helpers (used by the per-language lexers) ---

func wordsSet(s string) map[string]bool {
	m := make(map[string]bool)
	for w := range strings.FieldsSeq(s) {
		m[w] = true
	}
	return m
}

func isDigit(r rune) bool      { return r >= '0' && r <= '9' }
func isIdentStart(r rune) bool { return r == '_' || unicode.IsLetter(r) }
func isIdentPart(r rune) bool  { return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) }

// scanQuoted returns the index just past a single-line quoted run beginning at
// rs[start] (the opening quote). Honors backslash escapes; stops at the matching
// quote, a newline, or end of input.
func scanQuoted(rs []rune, start int) int {
	q := rs[start]
	j := start + 1
	for j < len(rs) {
		if rs[j] == '\\' && j+1 < len(rs) && rs[j+1] != '\n' {
			j += 2
			continue
		}
		if rs[j] == q || rs[j] == '\n' {
			break
		}
		j++
	}
	if j < len(rs) && rs[j] == q {
		j++
	}
	return j
}

// scanNumber returns the index just past a numeric literal beginning at rs[start]
// (a digit). Accepts hex/octal/binary prefixes, digit separators, and a dot.
func scanNumber(rs []rune, start int) int {
	j := start
	for j < len(rs) {
		r := rs[j]
		if isDigit(r) || r == '.' || r == '_' ||
			(r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') ||
			r == 'x' || r == 'X' || r == 'o' || r == 'O' || r == 'b' || r == 'B' {
			j++
			continue
		}
		break
	}
	return j
}

// nextNonSpaceIs reports whether the next non-space, non-tab rune at or after i
// equals c. Used for the "identifier followed by (" function heuristic.
func nextNonSpaceIs(rs []rune, i int, c rune) bool {
	for i < len(rs) {
		if rs[i] == ' ' || rs[i] == '\t' {
			i++
			continue
		}
		return rs[i] == c
	}
	return false
}

// nextNonSpaceIsIdent reports whether the next non-space, non-tab rune at or
// after i begins an identifier. Used to arm the "name after func/type" type
// heuristic only when a name actually follows (not '(' or '{').
func nextNonSpaceIsIdent(rs []rune, i int) bool {
	for i < len(rs) {
		if rs[i] == ' ' || rs[i] == '\t' {
			i++
			continue
		}
		return isIdentStart(rs[i])
	}
	return false
}

const operatorRunes = "{}()[]<>+-*/%=&|^~!?:;,."

func isOperator(r rune) bool { return strings.ContainsRune(operatorRunes, r) }