{{ message }}
Repository navigation
Expand file tree
/
Copy pathparser.ts
More file actions
384 lines (334 loc) · 11.8 KB
/
Copy pathparser.ts
File metadata and controls
384 lines (334 loc) · 11.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
import type { EditorState } from "@codemirror/state";
import type { AST, Option, Parser } from "node-sql-parser";
import { debug } from "../debug.js";
import { lazy } from "../utils.js";
import type { SqlParseError, SqlParseResult, SqlParser } from "./types.js";
export interface ParserOption extends Option {
database: SupportedDialects;
/**
* If true, the parser will quote brackets in the SQL query which will satisfy the parser.
* This is useful if you want to interpolate variables in f-strings.
*
* @example
* ```sql
* SELECT {id} -> SELECT '{id}'
* ```
*
* @experimental This is an experimental feature and may break parsing.
*/
ignoreBrackets?: boolean;
}
export interface NodeSqlParserOptions {
getParserOptions?: (state: EditorState) => ParserOption;
}
export interface NodeSqlParserResult extends SqlParseResult {
ast?: AST | AST[];
}
/**
* A SQL parser wrapper around node-sql-parser with enhanced error handling
* and validation capabilities for CodeMirror integration.
*
* @example Custom dialect
* ```ts
* import { NodeSqlParser } from "@marimo-team/codemirror-sql";
*
* const myParser = new NodeSqlParser({
* getParserOptions: (state) => ({
* dialect: getDialect(state),
* parseOptions: {
* includeLocations: true,
* },
* }),
* });
* ```
*/
export class NodeSqlParser implements SqlParser {
private opts: NodeSqlParserOptions;
private parser: Parser | null = null;
// Record the column number to the offset amount
private offsetRecord: Record<number, number> = {};
constructor(opts: NodeSqlParserOptions = {}) {
this.opts = opts;
}
/**
* Lazy import of the node-sql-parser package and create a new Parser instance.
*/
private getParser = lazy(async () => {
if (this.parser) {
return this.parser;
}
const module = await import("node-sql-parser");
// Support for ESM and CJS
const { Parser } = module.default || module;
this.parser = new Parser();
return this.parser;
});
async parse(sql: string, opts: { state: EditorState }): Promise<NodeSqlParserResult> {
// Reset the offset map on each parse
this.offsetRecord = {};
const parserOptions = this.opts.getParserOptions?.(opts.state);
const sanitizedSql = await this.sanitizeSql(sql, parserOptions);
try {
const parser = await this.getParser();
// Check if this is DuckDB dialect and apply custom processing
if (parserOptions?.database === "DuckDB") {
return this.parseWithDuckDBSupport(sanitizedSql, parserOptions);
}
const ast = parser.astify(sanitizedSql, parserOptions);
return {
success: true,
errors: [],
ast,
};
} catch (error: unknown) {
const parseError = this.extractErrorInfo(error, sanitizedSql);
return {
success: false,
errors: [parseError],
};
}
}
async sanitizeSql(sql: string, parserOptions?: ParserOption): Promise<string> {
if (parserOptions?.ignoreBrackets) {
const { sql: replacedSql, offsetRecord } = replaceBracketsWithQuotes(sql);
this.offsetRecord = offsetRecord;
return replacedSql;
}
return sql;
}
/**
* Parse SQL with DuckDB syntax support
* This is not robust, we catch main cases.
*/
private async parseWithDuckDBSupport(
sql: string,
parserOptions: Option,
): Promise<NodeSqlParserResult> {
const parser = await this.getParser();
// Remove comments and normalize for pattern checking
const sqlToCheck = removeCommentsFromStart(sql).trimStart().toLowerCase();
// Handle unsupported DuckDB syntax patterns
if (sqlToCheck.startsWith("from") || sqlToCheck.includes("macro")) {
debug("Unsupported DuckDB syntax");
return { success: true, errors: [] };
}
let modifiedSql = sql;
// Postgres does not support `CREATE OR REPLACE` for tables
if (sqlToCheck.includes("create or replace table")) {
const offset = "create or replace table".length - "create table".length;
this.offsetRecord[sql.indexOf("create or replace table")] = -offset;
modifiedSql = sql.replace(/create or replace table/i, "create table");
}
// Try standard parsing with PostgreSQL dialect
try {
const postgresOptions = { ...parserOptions, database: "PostgreSQL" };
const ast = parser.astify(modifiedSql, postgresOptions);
return { success: true, errors: [], ast };
} catch (error) {
// Use the original sql since we manually apply the offset
const parseError = this.extractErrorInfo(error, sql);
return { success: false, errors: [parseError] };
}
}
/**
* @param error - The error object
* @param sql - The SQL string
* @param offset - The offset to add to the column position. Default is 0.
* @returns The parsed error information
*/
private extractErrorInfo(error: unknown, sql: string): SqlParseError {
let line = 1;
let column = 1;
const message = (error as Error)?.message || "SQL parsing error";
const errorObj = error as {
location?: { start?: { line: number; column: number } };
hash?: { line: number; loc?: { first_column: number } };
};
if (errorObj?.location) {
line = errorObj.location.start?.line || 1;
column = errorObj.location.start?.column || 1;
} else if (errorObj?.hash) {
line = errorObj.hash.line || 1;
column = errorObj.hash.loc?.first_column || 1;
} else {
const lineMatch = message.match(/line (\d+)/i);
const columnMatch = message.match(/column (\d+)/i);
if (lineMatch?.[1]) {
line = parseInt(lineMatch[1], 10);
}
if (columnMatch?.[1]) {
column = parseInt(columnMatch[1], 10);
}
}
/**
* Map the error column back to the original (pre-replacement) SQL
* SELECT {id} FRO users
* ^ error position should be here
*
* SELECT '{id}' FRO users
* ^ parser reports this
* So in this case, we subtract the offset from the column position.
*
* offsetRecord keys are absolute offsets into the pre-replacement SQL,
* while the parser reports line-relative columns, so only replacements
* on the error's own line (and before the error column) shift it.
* Replacements never add or remove newlines, so line numbers match.
*/
const replacements = Object.entries(this.offsetRecord)
.map(([position, offset]) => ({ position: parseInt(position, 10), offset }))
.sort((a, b) => a.position - b.position);
let shift = 0;
for (const { position, offset } of replacements) {
const replacedPosition = position + shift;
shift += offset;
const beforeReplacement = sql.slice(0, replacedPosition);
const replacementLine = (beforeReplacement.match(/\n/g)?.length ?? 0) + 1;
const replacementColumn = replacedPosition - beforeReplacement.lastIndexOf("\n");
if (line === replacementLine && column > replacementColumn) {
column -= offset;
}
}
// Ensure we don't exceed the sql length
if (column > sql.length) {
column = sql.length;
}
return {
message: this.cleanErrorMessage(message),
line: Math.max(1, line),
column: Math.max(1, column),
severity: "error" as const,
};
}
private cleanErrorMessage(message: string): string {
return message
.replace(/^Error: /, "")
.replace(/Expected .* but .* found\./i, (match) =>
match.replace(/but .* found/, "found unexpected token"),
)
.trim();
}
async validateSql(sql: string, opts: { state: EditorState }): Promise<SqlParseError[]> {
const result = await this.parse(sql, opts);
return result.errors;
}
/**
* Extracts table references from a SQL query using node-sql-parser
* @param sql The SQL query to analyze
* @returns Array of table names referenced in the query
*/
async extractTableReferences(sql: string, opts?: { state: EditorState }): Promise<string[]> {
const parserOptions = opts ? this.opts.getParserOptions?.(opts.state) : undefined;
try {
const parser = await this.getParser();
const tableList = parser.tableList(sql, parserOptions);
// Clean up table names - node-sql-parser returns format like "select::null::users"
return tableList.map((table: string) => {
const parts = table.split("::");
return parts[parts.length - 1] || table;
});
} catch {
return [];
}
}
/**
* Extracts column references from a SQL query using node-sql-parser
* @param sql The SQL query to analyze
* @returns Array of column names referenced in the query
*/
async extractColumnReferences(sql: string, opts?: { state: EditorState }): Promise<string[]> {
const parserOptions = opts?.state ? this.opts.getParserOptions?.(opts.state) : undefined;
try {
const parser = await this.getParser();
const columnList = parser.columnList(sql, parserOptions);
// Clean up column names - node-sql-parser returns format like "select::null::users"
const cleanColumnList = columnList.map((column: string) => {
const parts = column.split("::");
return parts[parts.length - 1] || column;
});
return cleanColumnList;
} catch {
return [];
}
}
}
type CommentType = "--" | "/*";
function removeCommentsFromStart(sql: string, commentTypes: CommentType[] = ["/*", "--"]): string {
const regexPatterns: string[] = [];
// Multi-line comments
if (commentTypes.includes("/*")) {
regexPatterns.push("/\\*[\\s\\S]*?\\*/");
}
// Single-line comments
if (commentTypes.includes("--")) {
regexPatterns.push("--[^\\n]*");
}
if (regexPatterns.length === 0) return sql;
const commentRegex = new RegExp(`^\\s*(${regexPatterns.join("|")})\\s*`, "");
// Keep removing comments from the start until no more are found
let result = sql;
let prevResult = "";
while (result !== prevResult) {
prevResult = result;
result = result.replace(commentRegex, "");
}
return result;
}
const QUOTE_LENGTH = "''".length;
/**
* Replaces unquoted curly bracket expressions (e.g., {id}) with quoted strings (e.g., '{id}'),
* ignoring brackets already inside single or double quotes.
*
* Returns the modified SQL and a record mapping the index of each replaced bracket to the
* number of characters added (for offset tracking).
*
* @example
* replaceBracketsWithQuotes("SELECT {id}, '{name}' FROM users");
* // => {
* // sql: "SELECT '{id}', '{name}' FROM users",
* // offsetRecord: { 7: 2 }
* // }
*/
function replaceBracketsWithQuotes(sql: string): {
sql: string;
offsetRecord: Record<number, number>;
} {
const offsetRecord: Record<number, number> = {};
// Quote bodies use the unrolled-loop form (`[^"\\]*(?:\\.[^"\\]*)*`) instead
// of `(?:[^"\\]|\\.)*` to avoid polynomial backtracking on unclosed strings
const replacedSql = sql.replace(
/("[^"\\]*(?:\\.[^"\\]*)*")|('[^'\\]*(?:\\.[^'\\]*)*')|(\{[^}]*\})/g,
(match, doubleQuoted, singleQuoted, bracket, offset) => {
// If it's a quoted string, return it as-is
if (doubleQuoted || singleQuoted) {
return match;
}
// If it's a bracket, quote it and record the offset
if (bracket) {
offsetRecord[offset] = QUOTE_LENGTH;
return `'${bracket}'`;
}
return match;
},
);
return { sql: replacedSql, offsetRecord };
}
export const exportedForTesting = { replaceBracketsWithQuotes, removeCommentsFromStart };
/**
* https://github.com/taozhi8833998/node-sql-parser?tab=readme-ov-file#supported-database-sql-syntax
* While DuckDB is not supported in the library, we perform some special handling for it and treat it as PostgreSQL.
*/
export type SupportedDialects =
| "Athena"
| "BigQuery"
| "DB2"
| "Hive"
| "MariaDB"
| "MySQL"
| "PostgreSQL"
| "DuckDB"
| "Redshift"
| "Sqlite"
| "TransactSQL"
| "FlinkSQL"
| "Snowflake"
| "Noql";
You can’t perform that action at this time.