parser: fix CRLF(\r\n) file parsing error, SyntaxError: 'invalid syntax'

wetor · ncw · commit 652daef83367 · 2023-03-14T10:35:14.000Z
diff --git a/parser/lexer.go b/parser/lexer.go
@@ -110,6 +110,9 @@ func (x *yyLex) dequeue() int {
 func (x *yyLex) refill() {
 	var err error
 	x.line, err = x.reader.ReadString('\n')
+	if strings.HasSuffix(x.line, "\r\n") {
+		x.line = x.line[:len(x.line)-2] + "\n"
+	}
 	if yyDebug >= 2 {
 		fmt.Printf("line = %q, err = %v\n", x.line, err)
 	}
diff --git a/parser/lexer_test.go b/parser/lexer_test.go
@@ -262,7 +262,7 @@ func TestLex(t *testing.T) {
 		{"01", "illegal decimal with leading zero 1:0", "exec", LexTokens{
 			{FILE_INPUT, nil, ast.Pos{0, 0}},
 		}},
-		{"1\n 2\n  3\n4\n", "", "exec", LexTokens{
+		{"1\n 2\r\n  3\r\n4\n", "", "exec", LexTokens{
 			{FILE_INPUT, nil, ast.Pos{0, 0}},
 			{NUMBER, py.Int(1), ast.Pos{1, 0}},
 			{NEWLINE, nil, ast.Pos{1, 1}},
