Allow hooking into the filesystem and providing a persistent backend (like absurd-sql) by jlongster · Pull Request #481 · sql-js/sql.js · GitHub
Skip to content

Allow hooking into the filesystem and providing a persistent backend (like absurd-sql) #481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jlongster wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Allow hooking into the filesystem and providing a persistent backend (like absurd-sql) #481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
04da985
Implement changes required for IndexedDB-backed filesystem
jlongster Jul 21, 2021
1acb3f4
Improved exports for testing
jlongster Aug 2, 2021
ebc92a0
Add reset_filesystem
jlongster Aug 2, 2021
c1aea3e
Don't delete file on close
jlongster Aug 6, 2021
54ec387
Fix makefile fs-externs path
quolpr Aug 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions Makefile
View file
Open in desktop
67 changes: 61 additions & 6 deletions src/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,13 +811,20 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
* @memberof module:SqlJs
* Open a new database either by creating a new one or opening an existing
* one stored in the byte array passed in first argument
* @param {number[]} data An array of bytes representing
* an SQLite database file
* @param {number[]|string} data An array of bytes representing
* an SQLite database file or a path
* @param {Object} opts Options to specify a filename
*/
function Database(data) {
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
if (data != null) {
function Database(data, { filename = false } = {}) {
if(filename === false) {
this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0);
this.memoryFile = true;
if (data != null) {
FS.createDataFile("/", this.filename, data, true, true);
}
}
else {
this.filename = data;
}
this.handleError(sqlite3_open(this.filename, apiTemp));
this.db = getValue(apiTemp, "i32");
Expand Down Expand Up @@ -1103,7 +1110,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
Object.values(this.functions).forEach(removeFunction);
this.functions = {};
this.handleError(sqlite3_close_v2(this.db));
FS.unlink("/" + this.filename);

if(this.memoryFile) {
FS.unlink("/" + this.filename);
}
this.db = null;
};

Expand Down Expand Up @@ -1231,4 +1241,49 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {

// export Database to Module
Module.Database = Database;

// Because emscripten doesn't allow us to handle `ioctl`, we need
// to manually install lock/unlock methods. Unfortunately we need
// to keep track of a mapping of `sqlite_file*` pointers to filename
// so that we can tell our filesystem which files to lock/unlock
var sqliteFiles = new Map();

Module["register_for_idb"] = (customFS) => {
var SQLITE_BUSY = 5;

function open(namePtr, file) {
var path = UTF8ToString(namePtr);
sqliteFiles.set(file, path);
}

function lock(file, lockType) {
var path = sqliteFiles.get(file);
var success = customFS.lock(path, lockType)
return success? 0 : SQLITE_BUSY;
}

function unlock(file,lockType) {
var path = sqliteFiles.get(file);
customFS.unlock(path, lockType)
return 0;
}

let lockPtr = addFunction(lock, 'iii');
let unlockPtr = addFunction(unlock, 'iii');
let openPtr = addFunction(open, 'vii');
Module["_register_for_idb"](lockPtr, unlockPtr, openPtr)
}

// TODO: This isn't called from anywhere yet. We need to
// somehow cleanup closed files from `sqliteFiles`
Module["cleanup_file"] = (path) => {
let filesInfo = [...sqliteFiles.entries()]
let fileInfo = filesInfo.find(f => f[1] === path);
sqliteFiles.delete(fileInfo[0])
}

Module["reset_filesystem"] = () => {
FS.root = null;
FS.staticInit();
}
};
2 changes: 2 additions & 0 deletions src/exported_functions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,7 @@
"_sqlite3_result_int",
"_sqlite3_result_int64",
"_sqlite3_result_error",
"_sqlite3_vfs_find",
"_register_for_idb",
"_RegisterExtensionFunctions"
]
3 changes: 2 additions & 1 deletion src/exported_runtime_methods.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@
"stackAlloc",
"stackSave",
"stackRestore",
"UTF8ToString"
"UTF8ToString",
"FS"
]
48 changes: 48 additions & 0 deletions src/fs-externs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/**
* @externs
*/

Module.FS = class {
constructor() {
this.ErrnoError = class {};
}
mount() {}
isRoot() {}
isFile() {}
isDir() {}
stat() {}
/** @return {FSNode} */
lookupPath() {}
/** @return {FSNode} */
lookupNode() {}
/** @return {FSNode} */
createNode() {}
/** @return {FSNode} */
mknod() {}
};

Module.FS.FSNode = class {
constructor() {
this.node_ops = {
getattr: () => {},
setattr: () => {},
lookup: () => {},
mknod: () => {},
rename: () => {},
unlink: () => {},
rmdir: () => {},
reaaddir: () => {},
symlink: () => {},
readlink: () => {}
};

this.stream_ops = {
llseek: () => {},
read: () => {},
write: () => {},
allocate: () => {},
mmap: () => {},
msync: () => {}
};
}
};
47 changes: 47 additions & 0 deletions src/vfs.c
View file
Open in desktop