From c7266894590f1cd21c54b43f81ba36a7fc5204f4 Mon Sep 17 00:00:00 2001 From: Misode Date: Wed, 6 Nov 2024 06:31:59 +0100 Subject: [PATCH] Implement memory file system and mixed file system --- src/app/services/FileSystem.ts | 198 +++++++++++++++++++++++++++++++++ src/app/services/Spyglass.ts | 12 +- 2 files changed, 207 insertions(+), 3 deletions(-) diff --git a/src/app/services/FileSystem.ts b/src/app/services/FileSystem.ts index 2c6a44c8..d7bf6659 100644 --- a/src/app/services/FileSystem.ts +++ b/src/app/services/FileSystem.ts @@ -41,6 +41,204 @@ class BrowserEventEmitter implements core.ExternalEventEmitter { } } +export class MixedFileSystem implements core.ExternalFileSystem { + private watcher: MixedWatcher | undefined + + constructor( + public readonly base: core.ExternalFileSystem, + public readonly overlays: Map = new Map(), + ) {} + + async setOverlay(prefix: string, fileSystem: core.ExternalFileSystem) { + this.overlays.set(prefix, fileSystem) + if (this.watcher) { + await this.watcher.withOverlay(prefix, fileSystem) + } + return this + } + + getFileSystem(location: core.FsLocation) { + for (const [prefix, fs] of this.overlays.entries()) { + if (location.toString().startsWith(prefix)) { + return fs + } + } + return this.base + } + + chmod(location: core.FsLocation, mode: number) { + return this.getFileSystem(location).chmod(location, mode) + } + mkdir(location: core.FsLocation, options?: { mode?: number | undefined, recursive?: boolean | undefined } | undefined) { + return this.getFileSystem(location).mkdir(location, options) + } + readdir(location: core.FsLocation) { + return this.getFileSystem(location).readdir(location) + } + readFile(location: core.FsLocation) { + return this.getFileSystem(location).readFile(location) + } + showFile(path: core.FsLocation) { + return this.getFileSystem(path).showFile(path) + } + stat(location: core.FsLocation) { + return this.getFileSystem(location).stat(location) + } + unlink(location: core.FsLocation) { + return this.getFileSystem(location).unlink(location) + } + watch(locations: core.FsLocation[], options: { usePolling?: boolean | undefined }) { + this.watcher = new MixedWatcher(this, locations, options) + return this.watcher + } + writeFile(location: core.FsLocation, data: string | Uint8Array, options?: { mode: number } | undefined) { + return this.getFileSystem(location).writeFile(location, data, options) + } +} + +class MixedWatcher extends BrowserEventEmitter implements core.FsWatcher { + constructor( + fs: MixedFileSystem, + private readonly locations: core.FsLocation[], + private readonly options: { usePolling?: boolean | undefined }, + ) { + super() + Promise.all([ + this.initWatcher(fs.base, locations, options), + ...[...fs.overlays.values()].map(overlay => this.initWatcher(overlay, locations, options)), + ]).then(() => { + this.emit('ready') + }) + } + + private initWatcher(fs: core.ExternalFileSystem, locations: core.FsLocation[], options: { usePolling?: boolean | undefined }) { + return new Promise((res, rej) => { + fs.watch(locations, options) + .once('ready', () => res()) + .on('add', uri => this.emit('add', uri)) + .on('change', uri => this.emit('change', uri)) + .on('unlink', uri => this.emit('unlink', uri)) + .on('error', e => rej(e)) + }) + } + + withOverlay(_prefix: string, fs: core.ExternalFileSystem) { + return this.initWatcher(fs, this.locations, this.options) + } + + async close(): Promise {} +} + +export class MemoryFileSystem implements core.ExternalFileSystem { + private readonly states = new Map() + private watcher: MemoryWatcher | undefined + + async chmod(_location: core.FsLocation, _mode: number): Promise { + return + } + async mkdir( + location: core.FsLocation, + _options?: { mode?: number | undefined, recursive?: boolean | undefined } | undefined, + ): Promise { + location = core.fileUtil.ensureEndingSlash(location.toString()) + if (this.states.has(location)) { + throw new Error(`EEXIST: ${location}`) + } + this.states.set(location, { type: 'directory' }) + } + async readdir(location: core.FsLocation): Promise<{ name: string, isDirectory(): boolean, isFile(): boolean, isSymbolicLink(): boolean }[]> { + const result: { name: string, isDirectory(): boolean, isFile(): boolean, isSymbolicLink(): boolean }[] = [] + for (const [path, entry] of this.states.entries()) { + if (path.startsWith(location)) { + result.push({ + name: path, + isDirectory: () => entry.type === 'directory', + isFile: () => entry.type === 'file', + isSymbolicLink: () => false, + }) + } + } + return [] + } + async readFile(location: core.FsLocation): Promise { + location = location.toString() + const entry = this.states.get(location) + if (!entry) { + throw new Error(`ENOENT: ${location}`) + } else if (entry.type === 'directory') { + throw new Error(`EISDIR: ${location}`) + } + return entry.content + } + async showFile(_path: core.FsLocation): Promise { + throw new Error('showFile not supported on browser') + } + async stat(location: core.FsLocation): Promise<{ isDirectory(): boolean, isFile(): boolean }> { + location = location.toString() + const entry = this.states.get(location) + if (!entry) { + throw new Error(`ENOENT: ${location}`) + } + return { isDirectory: () => entry.type === 'directory', isFile: () => entry.type === 'file' } + } + async unlink(location: core.FsLocation): Promise { + location = location.toString() + const entry = this.states.get(location) + if (!entry) { + throw new Error(`ENOENT: ${location}`) + } + this.states.delete(location) + this.watcher?.tryEmit('unlink', location) + } + watch(locations: core.FsLocation[]): core.FsWatcher { + this.watcher = new MemoryWatcher(this.states, locations) + return this.watcher + } + async writeFile( + location: core.FsLocation, + data: string | Uint8Array, + _options?: { mode: number } | undefined, + ): Promise { + location = location.toString() + if (typeof data === 'string') { + data = new TextEncoder().encode(data) + } + const existed = this.states.has(location) + this.states.set(location, { type: 'file', content: data }) + if (existed) { + this.watcher?.tryEmit('change', location) + } else { + this.watcher?.tryEmit('add', location) + } + } +} + +class MemoryWatcher extends BrowserEventEmitter implements core.FsWatcher { + constructor( + states: Map, + private readonly locations: core.FsLocation[], + ) { + super() + setTimeout(() => { + for (const location of states.keys()) { + this.tryEmit('add', location) + } + this.emit('ready') + }) + } + + tryEmit(eventName: string, uri: string) { + for (const location of this.locations) { + if (uri.startsWith(location)) { + this.emit(eventName, uri) + break + } + } + } + + async close(): Promise {} +} + export class IndexedDbFileSystem implements core.ExternalFileSystem { public static readonly dbName = 'misode-spyglass-fs' public static readonly dbVersion = 1 diff --git a/src/app/services/Spyglass.ts b/src/app/services/Spyglass.ts index b2317d88..7919b7d9 100644 --- a/src/app/services/Spyglass.ts +++ b/src/app/services/Spyglass.ts @@ -17,7 +17,7 @@ import siteConfig from '../Config.js' import { computeIfAbsent, genPath, message } from '../Utils.js' import type { VersionMeta } from './DataFetcher.js' import { fetchBlockStates, fetchRegistries, fetchVanillaMcdoc, fetchVersions, getVersionChecksum } from './DataFetcher.js' -import { IndexedDbFileSystem } from './FileSystem.js' +import { IndexedDbFileSystem, MixedFileSystem } from './FileSystem.js' import type { VersionId } from './Versions.js' const builtinMcdoc = ` @@ -46,13 +46,14 @@ interface ClientDocument { } export class SpyglassClient { + public readonly fs = new MixedFileSystem(new IndexedDbFileSystem()) public readonly externals: core.Externals = { ...BrowserExternals, archive: { ...BrowserExternals.archive, decompressBall, }, - fs: new IndexedDbFileSystem(), + fs: this.fs, } public readonly documents = new Map() @@ -212,8 +213,11 @@ export class SpyglassService { const service = new core.Service({ logger, profilers: new core.ProfilerFactory(logger, [ + 'cache#load', + 'cache#save', 'project#init', 'project#ready', + 'project#ready#bind', ]), project: { cacheRoot: 'file:///cache/', @@ -266,7 +270,9 @@ export class SpyglassService { }, }) await service.project.ready() - await service.project.cacheService.save() + setTimeout(() => { + service.project.cacheService.save() + }, 10_000) return new SpyglassService(versionId, service, client) } }