Merge remote-tracking branch 'origin/main' into main · Java-Echo/DistributedMiniSQL@679fa08 · GitHub
Skip to content

Commit 679fa08

Browse files
Java-EchoJava-Echo
committed
Merge remote-tracking branch 'origin/main' into main
2 parents 9b7ab61 + 4865c54 commit 679fa08

22 files changed

Lines changed: 5199 additions & 1699 deletions

‎main.go‎

Lines changed: 2 additions & 2 deletions

‎src/Interpreter/lexer/lexer.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package lexer
2+
3+
import (
4+
"io"
5+
"log"
6+
)
7+
8+
type Token int
9+
10+
// 工具类：接受由Scanner传入的经过基本处理的token，通过lookahead来赋予该token更多信息
11+
type Tokenizer interface {
12+
FromStrLit(lit string, TokenType Token, lastToken int) int
13+
}
14+
15+
// 工具类：初步处理输入，得到基础的token
16+
type Scanner interface {
17+
Scan() (tok Token, lit string)
18+
}
19+
20+
// 模块类：将输入解析为token的总体工具类
21+
type LexerImpl struct {
22+
scanner Scanner // 处理输入的工具类
23+
tokenizer Tokenizer // 进一步赋予token信息的工具类
24+
Result interface{}
25+
}
26+
27+
// 存储类：存储token解析的结果，最终的Lex()主要利用这个对象来返回结果
28+
type LexerResult struct {
29+
Token int
30+
Literal string
31+
}
32+
33+
// 新建一个“将输入解析为token”的总体工具类的实例化对象
34+
func NewLexerImpl(r io.Reader) *LexerImpl {
35+
return &LexerImpl{
36+
scanner: NewScanner(r),
37+
tokenizer: NewTokenizer(),
38+
}
39+
}
40+
41+
func (li *LexerImpl) Lex(lastToken int) (*LexerResult, error) {
42+
result := &LexerResult{}
43+
44+
tok, lit := li.scanner.Scan() // 这里的scanner已经完成了输入流的基础token化
45+
46+
switch tok {
47+
case T_EOF:
48+
// Stop lex
49+
case T_IDENT, T_INTEGER, T_FLOAT, T_STRING, T_LEFT_PARENTHESIS, T_RIGHT_PARENTHESIS, T_COMMA, T_SEMICOLON, T_EQUAL, T_ANGLE_LEFT, T_ANGLE_RIGHT, T_ANGLE_LEFT_EQUAL, T_ANGLE_RIGHT_EQUAL, T_NOT_EQUAL, T_ASTERISK, T_POINT:
50+
result.Literal = lit
51+
default:
52+
log.Printf("UnexpectedToken: tok is %d, lit is %s\n", tok, lit)
53+
// return nil, UnexpectedTokenErr // ToDo:这里的错误机制需要完善
54+
return nil, nil
55+
}
56+
57+
result.Token = li.tokenizer.FromStrLit(lit, tok, lastToken)
58+
59+
return result, nil
60+
}

‎src/Interpreter/lexer/lexer_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package lexer
2+
3+
import (
4+
"log"
5+
"strings"
6+
"testing"
7+
)
8+
9+
var sql_strings = []string{
10+
// "create table 1_a",
11+
" ",
12+
"create table cxz(" +
13+
"afsdfsad int unique," +
14+
"what char(30) not null," +
15+
"primary key (what)" +
16+
");",
17+
"select a,b,c,d,e,f,g from cxz where a=123 and b=456 or c=234;",
18+
}
19+
20+
//DoTo:这里有个bug就是我们无法正确地停止输入
21+
func TestLexerLex(t *testing.T) {
22+
for _, str := range sql_strings {
23+
LastToken := 0
24+
io := strings.NewReader(str) // 组装io
25+
impl := NewLexerImpl(io) // 组装待测试的LexerImpl
26+
for LastToken != int(T_EOF) {
27+
r, _ := impl.Lex(LastToken)
28+
29+
tokVal := r.Token
30+
literal := r.Literal
31+
LastToken = tokVal
32+
33+
// log.Print(tokVal)
34+
// log.Print(" ")
35+
log.Print(literal)
36+
// log.Print(" ")
37+
// log.Print(LastToken)
38+
if tokVal == 0 {
39+
// 检测是否到达输入末尾
40+
break
41+
}
42+
}
43+
44+
}
45+
t.Errorf("不要慌，我只是想要一个log而已\n")
46+
}

src/parser/scanner.go renamed to src/Interpreter/lexer/scanner.go

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
package parser
1+
package lexer
22

33
import (
44
"bufio"
55
"bytes"
66
"io"
77
)
88

9-
type Token int
9+
// type Token int
1010

1111
// 词法分析中的初步结果(部分内容可以经过tokenize来形成语义更加明确的token)
1212
const (
1313
// 特殊标记
14-
ILLEGAL Token = iota
15-
EOF
14+
T_ILLEGAL Token = iota
15+
T_EOF
1616
WS // 空白字符
1717
// 常规类型数据
18-
IDENT // ID，此时我们并不区分关键词，而是归类到同一类
19-
INTEGER // 整数
20-
FLOAT // 浮点数
21-
STRING // 字符串
18+
T_IDENT // ID，此时我们并不区分关键词，而是归类到同一类
19+
T_INTEGER // 整数
20+
T_FLOAT // 浮点数
21+
T_STRING // 字符串
2222
// 其他标记
23-
ASTERISK // *
24-
COMMA // ,
25-
LEFT_PARENTHESIS // (
26-
RIGHT_PARENTHESIS // )
27-
SEMICOLON // ;
28-
EQUAL // =
29-
ANGLE_LEFT // <
30-
ANGLE_LEFT_EQUAL //<=
31-
ANGLE_RIGHT_EQUAL //>=
32-
ANGLE_RIGHT // >
33-
NOT_EQUAL // <> or !=
34-
POINT // .
23+
T_ASTERISK // *
24+
T_COMMA // ,
25+
T_LEFT_PARENTHESIS // (
26+
T_RIGHT_PARENTHESIS // )
27+
T_SEMICOLON // ;
28+
T_EQUAL // =
29+
T_ANGLE_LEFT // <
30+
T_ANGLE_LEFT_EQUAL //<=
31+
T_ANGLE_RIGHT_EQUAL //>=
32+
T_ANGLE_RIGHT // >
33+
T_NOT_EQUAL // <> or !=
34+
T_POINT // .
3535
)
3636

3737
type State int // 状态机的状态
@@ -77,7 +77,7 @@ func (s *InputScanner) Scan() (tok Token, lit string) {
7777
state := STATE_INIT
7878
for state != STATE_END {
7979
if checkCharType(ch) == ILLEGAL_SYMBOL {
80-
return ILLEGAL, string(ch)
80+
return T_ILLEGAL, string(ch)
8181
}
8282
// buf.WriteRune(ch)
8383
switch state {
@@ -92,21 +92,21 @@ func (s *InputScanner) Scan() (tok Token, lit string) {
9292
case SPECIAL_SYMBOL:
9393
switch ch {
9494
case eof:
95-
return EOF, ""
95+
return T_EOF, ""
9696
case '.':
97-
return POINT, string(ch)
97+
return T_POINT, string(ch)
9898
case '*':
99-
return ASTERISK, string(ch)
99+
return T_ASTERISK, string(ch)
100100
case ',':
101-
return COMMA, string(ch)
101+
return T_COMMA, string(ch)
102102
case '(':
103-
return LEFT_PARENTHESIS, string(ch)
103+
return T_LEFT_PARENTHESIS, string(ch)
104104
case ')':
105-
return RIGHT_PARENTHESIS, string(ch)
105+
return T_RIGHT_PARENTHESIS, string(ch)
106106
case ';':
107-
return SEMICOLON, string(ch)
107+
return T_SEMICOLON, string(ch)
108108
case '=':
109-
return EQUAL, string(ch)
109+
return T_EQUAL, string(ch)
110110
case '<':
111111
buf.WriteRune(ch)
112112
state = STATE_ANGLE_LEFT
@@ -116,22 +116,22 @@ func (s *InputScanner) Scan() (tok Token, lit string) {
116116
}
117117
case SPACE:
118118
case UNDERLINE:
119-
return ILLEGAL, string(ch)
119+
return T_ILLEGAL, string(ch)
120120
}
121121
case STATE_INTEGER:
122122
switch checkCharType(ch) {
123123
case NUM:
124124
buf.WriteRune(ch)
125125
case CHAR, SPACE, UNDERLINE:
126126
s.unread()
127-
return INTEGER, buf.String()
127+
return T_INTEGER, buf.String()
128128
case SPECIAL_SYMBOL:
129129
if ch == '.' {
130130
buf.WriteRune(ch)
131131
state = STATE_POINT
132132
} else {
133133
s.unread()
134-
return INTEGER, buf.String()
134+
return T_INTEGER, buf.String()
135135
}
136136
}
137137
case STATE_POINT:
@@ -140,63 +140,63 @@ func (s *InputScanner) Scan() (tok Token, lit string) {
140140
buf.WriteRune(ch)
141141
state = STATE_FRACTION
142142
case CHAR, SPECIAL_SYMBOL, SPACE, UNDERLINE:
143-
return ILLEGAL, string(ch)
143+
return T_ILLEGAL, string(ch)
144144
}
145145
case STATE_FRACTION:
146146
switch checkCharType(ch) {
147147
case NUM:
148148
buf.WriteRune(ch)
149149
case CHAR, SPECIAL_SYMBOL, SPACE, UNDERLINE:
150150
s.unread()
151-
return FLOAT, buf.String()
151+
return T_FLOAT, buf.String()
152152
}
153153
case STATE_IDENT:
154154
switch checkCharType(ch) {
155155
case NUM, CHAR, UNDERLINE:
156156
buf.WriteRune(ch)
157157
case SPECIAL_SYMBOL, SPACE:
158158
s.unread()
159-
return IDENT, buf.String()
159+
return T_IDENT, buf.String()
160160
}
161161
case STATE_ANGLE_LEFT:
162162
switch checkCharType(ch) {
163163
case NUM, CHAR, SPACE:
164164
s.unread()
165-
return ANGLE_LEFT, buf.String()
165+
return T_ANGLE_LEFT, buf.String()
166166
case SPECIAL_SYMBOL:
167167
// ch = s.read()
168168
if ch == '=' {
169-
return ANGLE_LEFT_EQUAL, "<="
169+
return T_ANGLE_LEFT_EQUAL, "<="
170170
} else if ch == '>' {
171-
return NOT_EQUAL, "<>"
171+
return T_NOT_EQUAL, "<>"
172172
} else {
173173
s.unread()
174-
return ANGLE_LEFT, buf.String()
174+
return T_ANGLE_LEFT, buf.String()
175175
}
176176
}
177177
case STATE_ANGLE_RIGHT:
178178
switch checkCharType(ch) {
179179
case NUM, CHAR, SPACE:
180180
s.unread()
181-
return ANGLE_RIGHT, buf.String()
181+
return T_ANGLE_RIGHT, buf.String()
182182
case SPECIAL_SYMBOL:
183183
// ch = s.read()
184184
if ch == '=' {
185-
return ANGLE_RIGHT_EQUAL, ">="
185+
return T_ANGLE_RIGHT_EQUAL, ">="
186186
} else {
187187
s.unread()
188-
return ANGLE_RIGHT, buf.String()
188+
return T_ANGLE_RIGHT, buf.String()
189189
}
190190
}
191191
}
192192
ch = s.read()
193193
}
194194

195-
return ILLEGAL, string(ch)
195+
return T_ILLEGAL, string(ch)
196196
}
197197

198198
// read reads the next rune from the buffered reader.
199-
// Returns the rune(0) if an error occurs (or io.EOF is returned).
199+
// Returns the rune(0) if an error occurs (or io.T_EOF is returned).
200200
func (s *InputScanner) read() rune {
201201
ch, _, err := s.r.ReadRune()
202202
if err != nil {

‎src/Interpreter/lexer/scanner_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package lexer
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestScan(t *testing.T) {
8+
// ToDo:现在我是懒得写了
9+
}

‎src/Interpreter/lexer/temp.go‎

Lines changed: 62 additions & 0 deletions

0 commit comments

Comments
 (0)