From 5c066749e253dbc90f1e542327b7c19fd1edcd61 Mon Sep 17 00:00:00 2001 From: Filip Stanis Date: Fri, 28 Aug 2026 16:11:07 -0400 Subject: [PATCH 1/2] workerfs --- .gitignore | 4 ++ README.md | 8 ++++ emscriptenbuild/build.sh | 11 ++++++ libgit2patchedfiles/examples/lg2.c | 11 ++++++ package.json | 1 + test-browser-workerfs/test.spec.js | 51 ++++++++++++++++++++++++++ test-browser-workerfs/worker.js | 59 ++++++++++++++++++++++++++++++ web-test-runner-workerfs.config.js | 30 +++++++++++++++ 8 files changed, 175 insertions(+) create mode 100644 test-browser-workerfs/test.spec.js create mode 100644 test-browser-workerfs/worker.js create mode 100644 web-test-runner-workerfs.config.js diff --git a/.gitignore b/.gitignore index a8c6c277..0d919273 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ lg2_opfs_async.js lg2_opfs_async.wasm lg2_opfs_jspi.js lg2_opfs_jspi.wasm +lg2_workerfs.js +lg2_workerfs.wasm +test-browser-workerfs/lg2_workerfs.js +test-browser-workerfs/lg2_workerfs.wasm test-browser-opfs-noniso/lg2_opfs_auto.js test-browser-opfs-noniso/lg2_opfs.js test-browser-opfs-noniso/lg2_opfs.wasm diff --git a/README.md b/README.md index b8c6a84c..4b0988a9 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ The files can also be loaded from public CDNs such as unpkg or jsDelivr (e.g. `h | Variant | File | Where it runs | Persistence | Notes | |---------|------|---------------|-------------|-------| | **Sync** | `lg2.js` | Browser: **Web Worker only**. Node.js: main thread or [worker_threads](https://nodejs.org/api/worker_threads.html) | MEMFS / [IDBFS](https://emscripten.org/docs/api_reference/Filesystem-API.html#filesystem-api-idbfs) / NODEFS | Smallest binary; synchronous `callMain`. Needs a worker in the browser because of synchronous HTTP and long-running operations. | +| **Sync (WORKERFS)** | `lg2_workerfs.js` | Browser: **Web Worker only** | MEMFS / WORKERFS (read-only mounts of Blobs/Files) | Minimal sync build with only MEMFS + WORKERFS (`./build.sh Release-workerfs`); no IDBFS/NODEFS. Mount read-only mirrors (e.g. a folder picked with the File System Access API) without copying, and keep the git index writable with lg2's `--index-file` flag. | | **Async** | `lg2_async.js` | Browser main thread or worker; Node.js | MEMFS / IDBFS / NODEFS | [Asyncify](https://emscripten.org/docs/porting/asyncify.html) build: `await lg.callMain(...)`. Larger binary, simplest client code. | | **OPFS (pthreads)** | `lg2_opfs.js` | Web Worker, requires [cross-origin isolation](https://developer.mozilla.org/en-US/docs/Web/API/Window/crossOriginIsolated) | [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) | Fastest OPFS variant; needs COOP/COEP headers for SharedArrayBuffer. | | **OPFS (JSPI)** | `lg2_opfs_jspi.js` | Web Worker, no isolation needed | OPFS | SAB-free, smallest OPFS binary; needs a [JSPI](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Suspending)-capable browser. | @@ -65,6 +66,7 @@ Complete working examples for every variant are in the test folders: - [test](./test/) for Node.js - [test-browser](./test-browser/) for the sync version in a web worker +- [test-browser-workerfs](./test-browser-workerfs/) for the WORKERFS sync variant - [test-browser-async](./test-browser-async/) for the async version - [test-browser-opfs](./test-browser-opfs/) for the OPFS (pthreads/WASMFS) version - [test-browser-opfs-noniso](./test-browser-opfs-noniso/) for the SAB-free OPFS variants (ASYNCIFY + JSPI) and the runtime loader @@ -210,6 +212,12 @@ Wasm-git supports multiple filesystem backends for different use cases: - **Build target**: Default (`./build.sh Release`) - **Platform**: Node.js only +### WORKERFS (Web Worker File System) +- **Use case**: Read-only mounts of Blobs/Files (e.g. a folder picked with the File System Access API) without copying their contents into memory +- **Build target**: `./build.sh Release-workerfs` → `lg2_workerfs.js` (links only `-lworkerfs.js -lmemfs.js`; no IDBFS/NODEFS) +- **Browser support**: Web Workers only (uses `FileReaderSync`) +- **Note**: Mounts are read-only. Redirect the git index to MEMFS with lg2's `--index-file ` argument — passed before the git command like `--git-dir`, e.g. `lg.callMain(['--index-file', '/gitindex', 'status'])` — so index-writing commands work against a read-only repository mount. + ### OPFS (Origin Private File System) - **Use case**: Modern browser persistent storage with better performance and quota management - **Browser support**: Chrome 86+, Edge 86+, Firefox 111+, Safari 15.2+ (JSPI variant: Chromium-based browsers with JSPI) diff --git a/emscriptenbuild/build.sh b/emscriptenbuild/build.sh index 92cb2dbe..54fa5f04 100755 --- a/emscriptenbuild/build.sh +++ b/emscriptenbuild/build.sh @@ -100,6 +100,17 @@ elif [ "$1" == "Debug-opfs-jspi" ]; then FS_LIBRARIES="" FS_EXPORTS="'FS','callMain','HEAPU8','ccall'" EXTRA_CMAKE_DEFINES="-DUSE_THREADS=OFF -DUSE_NSEC=OFF" +elif [ "$1" == "Release-workerfs" ]; then + BUILD_TYPE=Release + EXTRA_CMAKE_C_FLAGS="-Oz" + FS_LIBRARIES="-lworkerfs.js -lmemfs.js" + FS_EXPORTS="'FS','MEMFS','WORKERFS','callMain','HEAPU8'" + export LG2_OUTPUT_NAME=lg2_workerfs +elif [ "$1" == "Debug-workerfs" ]; then + BUILD_TYPE=Debug + FS_LIBRARIES="-lworkerfs.js -lmemfs.js" + FS_EXPORTS="'FS','MEMFS','WORKERFS','callMain','HEAPU8'" + export LG2_OUTPUT_NAME=lg2_workerfs fi # Before building, remove any ../libgit2/src/ transports/emscriptenhttp.c left from running setup.sh diff --git a/libgit2patchedfiles/examples/lg2.c b/libgit2patchedfiles/examples/lg2.c index 3152eda9..a436d299 100644 --- a/libgit2patchedfiles/examples/lg2.c +++ b/libgit2patchedfiles/examples/lg2.c @@ -72,6 +72,7 @@ int main(int argc, char **argv) struct args_info args = ARGS_INFO_INIT; git_repository *repo = NULL; const char *git_dir = NULL; + const char *index_file = NULL; int return_code = 1; size_t i; @@ -88,6 +89,8 @@ int main(int argc, char **argv) break; } else if (optional_str_arg(&git_dir, &args, "--git-dir", ".git")) { continue; + } else if (optional_str_arg(&index_file, &args, "--index-file", NULL)) { + continue; } else if (match_arg_separator(&args)) { break; } @@ -110,6 +113,14 @@ int main(int argc, char **argv) if (commands[i].requires_repo) { check_lg2(git_repository_open_ext(&repo, git_dir, 0, NULL), "Unable to open repository '%s'", git_dir); + + // redirect the index e.g. onto a writable MEMFS path + if (index_file) { + git_index *index = NULL; + check_lg2(git_index_open(&index, index_file), + "Unable to open index '%s'", index_file); + git_repository_set_index(repo, index); + } } return_code = run_command(commands[i].fn, repo, args); diff --git a/package.json b/package.json index bc4bd80c..250d2f57 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test": "mocha --exit test/**/*.spec.js", "test-browser": "wtr test-browser/**/*.spec.js", "test-browser-watch": "wtr test-browser/**/*.spec.js --watch", + "test-browser-workerfs": "wtr --config web-test-runner-workerfs.config.js", "test-browser-async": "wtr test-browser-async/**/*.spec.js", "test-browser-opfs": "wtr --config web-test-runner-opfs.config.js", "test-browser-opfs-noniso": "wtr --config web-test-runner-opfs-noniso.config.js", diff --git a/test-browser-workerfs/test.spec.js b/test-browser-workerfs/test.spec.js new file mode 100644 index 00000000..9eef999c --- /dev/null +++ b/test-browser-workerfs/test.spec.js @@ -0,0 +1,51 @@ +describe('wasm-git workerfs variant', function () { + this.timeout(20000); + + let worker; + + const createWorker = async () => { + worker = new Worker(new URL('worker.js', import.meta.url), { type: 'module' }); + await new Promise(resolve => { + worker.onmessage = msg => { + if (msg.data.ready) { + resolve(msg); + } + } + }); + }; + + const callWorker = async (command, params) => { + return await new Promise(resolve => { + worker.onmessage = msg => resolve(msg.data); + worker.postMessage(Object.assign({ + command: command + }, params)); + }); + }; + + this.afterAll(async () => { + worker.terminate(); + }); + + it('should get ready message from web worker', async () => { + await createWorker(); + }); + + it('should mount a blob with WORKERFS and read it back', async () => { + const result = await callWorker('mountworkerfs', { + filename: 'blobfile.txt', + contents: 'hello from a workerfs blob' + }); + assert(!result.stderr, `workerfs mount failed: ${result.stderr}`); + assert.equal(result.filecontents, 'hello from a workerfs blob'); + assert(result.dircontents.includes('blobfile.txt')); + }); + + it('should keep the git index on MEMFS with --index-file', async () => { + const result = await callWorker('initandadd'); + assert(!result.stderr, `git add failed: ${result.stderr}`); + assert.equal(result.exitcode, 0); + assert(result.indexOnMemfs, 'index was not written to /gitindex'); + assert(result.noIndexInRepo, '.git/index should not be written when --index-file is used'); + }); +}); diff --git a/test-browser-workerfs/worker.js b/test-browser-workerfs/worker.js new file mode 100644 index 00000000..8f4c22ec --- /dev/null +++ b/test-browser-workerfs/worker.js @@ -0,0 +1,59 @@ +let stdout = []; +let stderr = []; + +globalThis.wasmGitModuleOverrides = { + 'print': (text) => { + console.log(text); + stdout.push(text) + }, + 'printErr': (text) => { + console.error(text); + stderr.push(text); + } +}; + +const lg2mod = await import(new URL('lg2_workerfs.js', import.meta.url)); +const lg = await lg2mod.default(); + +const FS = lg.FS; +const WORKERFS = lg.WORKERFS; + +onmessage = (msg) => { + stderr = []; + stdout = []; + if (msg.data.command === 'mountworkerfs') { + try { + FS.mkdir('/workerfs'); + FS.mount(WORKERFS, { + blobs: [{ name: msg.data.filename, data: new Blob([msg.data.contents]) }] + }, '/workerfs'); + postMessage({ + filename: msg.data.filename, + filecontents: FS.readFile(`/workerfs/${msg.data.filename}`, { encoding: 'utf8' }), + dircontents: FS.readdir('/workerfs') + }); + } catch (e) { + postMessage({ 'stderr': JSON.stringify(e) }); + } + } else if (msg.data.command === 'initandadd') { + // git operations in MEMFS with the index redirected through --index-file + FS.mkdir('/workerfstestrepo'); + FS.chdir('/workerfstestrepo'); + lg.callMain(['init', '.']); + FS.writeFile('/workerfstestrepo/newfile.txt', 'hello workerfs variant'); + const code = lg.callMain(['--index-file', '/gitindex', 'add', 'newfile.txt']); + postMessage({ + exitcode: code, + indexOnMemfs: FS.analyzePath('/gitindex').exists, + noIndexInRepo: !FS.analyzePath('/workerfstestrepo/.git/index').exists, + stagedFiles: stdout.join('\n'), + stderr: stderr.join('\n') + }); + } else { + const args = msg.data.args || []; + lg.callMain([msg.data.command, ...args]); + postMessage({ stdout: stdout.join('\n'), stderr: stderr.join('\n'), }); + } +}; + +postMessage({ 'ready': true }); diff --git a/web-test-runner-workerfs.config.js b/web-test-runner-workerfs.config.js new file mode 100644 index 00000000..cf048283 --- /dev/null +++ b/web-test-runner-workerfs.config.js @@ -0,0 +1,30 @@ +import { playwrightLauncher } from '@web/test-runner-playwright'; + +export default { + files: [ + 'test-browser-workerfs/**/*.spec.js', + '!./node_modules/**/*', + ], + concurrency: 1, + watch: false, + testFramework: { + config: { + ui: 'bdd', + timeout: '5000', + }, + }, + testRunnerHtml: testRunnerImport => + ` + + + + + `, + browsers: [ + playwrightLauncher({ product: 'chromium' }), + ], +}; From 06d07a6803752637c83bb65b42d17df595ef3d4a Mon Sep 17 00:00:00 2001 From: Filip Stanis Date: Fri, 28 Aug 2026 16:24:59 -0400 Subject: [PATCH 2/2] ci --- .github/workflows/main.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 1 + package.json | 2 ++ preparepublishnpm.sh | 2 ++ 4 files changed, 37 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e07be13..1e0437dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,6 +68,34 @@ jobs: cd .. set -e npm run test-browser-async + workerfs: + name: WORKERFS + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - name: WORKERFS Build and Test + run: | + npm install + npx playwright install-deps + npx playwright install + sh setup.sh + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + ./emsdk install 6.0.3 + ./emsdk activate 6.0.3 + cd .. + source ./emsdk/emsdk_env.sh + cd emscriptenbuild + ./build.sh Release-workerfs + cd .. + ln -s ../emscriptenbuild/libgit2/examples/lg2_workerfs.js test-browser-workerfs/lg2_workerfs.js + ln -s ../emscriptenbuild/libgit2/examples/lg2_workerfs.wasm test-browser-workerfs/lg2_workerfs.wasm + set -e + npm run test-browser-workerfs opfs: name: OPFS runs-on: ubuntu-latest @@ -182,6 +210,7 @@ jobs: cd emscriptenbuild ./build.sh Release ./build.sh Release-async + ./build.sh Release-workerfs ./build.sh Release-opfs ./build.sh Release-opfs-async ./build.sh Release-opfs-jspi @@ -200,6 +229,8 @@ jobs: ln -sf ../lg2.wasm test-browser/lg2.wasm ln -sf ../lg2_async.js test-browser-async/lg2_async.js ln -sf ../lg2_async.wasm test-browser-async/lg2_async.wasm + ln -sf ../lg2_workerfs.js test-browser-workerfs/lg2_workerfs.js + ln -sf ../lg2_workerfs.wasm test-browser-workerfs/lg2_workerfs.wasm ln -sf ../lg2_opfs.js test-browser-opfs/lg2_opfs.js ln -sf ../lg2_opfs.wasm test-browser-opfs/lg2_opfs.wasm ln -sf ../../lg2_opfs.js examples/opfs/lg2_opfs.js @@ -209,6 +240,7 @@ jobs: done npm run test-browser npm run test-browser-async + npm run test-browser-workerfs npm run test-browser-opfs npm run test-opfs-example npm run test-browser-opfs-noniso diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ffc9afcb..ef9bdefd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,7 @@ jobs: cd emscriptenbuild ./build.sh Release ./build.sh Release-async + ./build.sh Release-workerfs ./build.sh Release-opfs ./build.sh Release-opfs-async ./build.sh Release-opfs-jspi diff --git a/package.json b/package.json index 250d2f57..74fb34dc 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "lg2_opfs_async.wasm", "lg2_opfs_jspi.js", "lg2_opfs_jspi.wasm", + "lg2_workerfs.js", + "lg2_workerfs.wasm", "lg2_opfs_auto.js", "README.md" ], diff --git a/preparepublishnpm.sh b/preparepublishnpm.sh index 39eba350..5216e778 100755 --- a/preparepublishnpm.sh +++ b/preparepublishnpm.sh @@ -11,5 +11,7 @@ cp emscriptenbuild/libgit2/examples/lg2_opfs_async.wasm . cp emscriptenbuild/libgit2/examples/lg2_opfs_async.js . cp emscriptenbuild/libgit2/examples/lg2_opfs_jspi.wasm . cp emscriptenbuild/libgit2/examples/lg2_opfs_jspi.js . +cp emscriptenbuild/libgit2/examples/lg2_workerfs.wasm . +cp emscriptenbuild/libgit2/examples/lg2_workerfs.js . # lg2_opfs_auto.js is a hand-written source file already at the repo root echo "npm package prepared"