From e69559f9752ed3599ead0020fffa520a022333cf Mon Sep 17 00:00:00 2001 From: Misode Date: Wed, 30 Dec 2020 18:58:54 +0100 Subject: [PATCH] Use Cache API instead of LocalStorage (#128) * Use Cache API instead of LocalStorage * Remove old cached data --- src/app/App.ts | 9 +- src/app/DataFetcher.ts | 194 ++++++++++++++----------- src/app/components/panels/TreePanel.ts | 5 +- src/app/hooks/customValidation.ts | 4 +- src/app/hooks/renderHtml.ts | 2 +- src/config.json | 21 ++- 6 files changed, 132 insertions(+), 103 deletions(-) diff --git a/src/app/App.ts b/src/app/App.ts index 8414a696..8ff7640a 100644 --- a/src/app/App.ts +++ b/src/app/App.ts @@ -40,7 +40,7 @@ export const Models: { config.models.filter(m => m.schema) .forEach(m => Models[m.id] = new DataModel(ObjectNode({}))) -export let BlockStateRegistry: { +export type BlockStateRegistry = { [block: string]: { properties: { [key: string]: string[] @@ -49,7 +49,7 @@ export let BlockStateRegistry: { [key: string]: string } } -} = {} +} export const App = { version: new LocalStorageProperty('schema_version', config.versions[config.versions.length - 1].id) @@ -70,7 +70,8 @@ export const App = { localesLoaded: new Property(false), loaded: new Property(false), mobilePanel: new Property('tree'), - settings: new Settings('generator_settings') + settings: new Settings('generator_settings'), + blockStateRegistry: {} as BlockStateRegistry } App.version.watchRun(async (value) => { @@ -108,7 +109,7 @@ App.mobilePanel.watchRun((value) => { }) async function updateSchemas(version: string) { - BlockStateRegistry = {} + App.blockStateRegistry = {} const collections = Versions[version].getCollections() App.collections.set(collections) await fetchData(collections, version) diff --git a/src/app/DataFetcher.ts b/src/app/DataFetcher.ts index 5d73231c..65ff4358 100644 --- a/src/app/DataFetcher.ts +++ b/src/app/DataFetcher.ts @@ -1,136 +1,130 @@ import { CollectionRegistry } from '@mcschema/core' -import { BlockStateRegistry, checkVersion } from './App' +import { App, BlockStateRegistry, checkVersion } from './App' import config from '../config.json' -const CACHE_FORMAT = 1 +['1.15', '1.16', '1.17'].forEach(v => localStorage.removeItem(`cache_${v}`)) -type VersionConfig = { +const CACHE_NAME = `misode-v1` + +type VersionRef = 'mcdata_master' | 'vanilla_datapack_summary' | 'vanilla_datapack_data' + +type Version = { id: string, - mcdata_ref: string, - vanilla_datapack_data_ref?: string, - vanilla_datapack_summary_ref?: string + refs: Partial<{ [key in VersionRef]: string }>, + dynamic?: boolean, } -const localStorageCache = (version: string) => `cache_${version}` declare var __MCDATA_MASTER_HASH__: string; declare var __VANILLA_DATAPACK_SUMMARY_HASH__: string; const mcdataUrl = 'https://raw.githubusercontent.com/Arcensoth/mcdata' const vanillaDatapackUrl = 'https://raw.githubusercontent.com/SPGoding/vanilla-datapack' -export const fetchData = async (target: CollectionRegistry, versionId: string) => { - const version = config.versions.find(v => v.id === versionId) - if (!version) return +const refs: { + id: VersionRef, + hash: string, + url: string +}[] = [ + { + id: 'mcdata_master', + hash: __MCDATA_MASTER_HASH__, + url: mcdataUrl + }, + { + id: 'vanilla_datapack_summary', + hash: __VANILLA_DATAPACK_SUMMARY_HASH__, + url: vanillaDatapackUrl + }, +] - const cache = JSON.parse(localStorage.getItem(localStorageCache(versionId)) ?? '{}') - const mcdataCacheValid = cache.format === CACHE_FORMAT && (version.mcdata_ref !== 'master' || cache.mcdata_hash === __MCDATA_MASTER_HASH__) - const vanillaDatapackCacheValid = cache.format === CACHE_FORMAT && (version.vanilla_datapack_summary_ref !== 'summary' || cache.vanilla_datapack_summary_hash === __VANILLA_DATAPACK_SUMMARY_HASH__) +export async function fetchData(target: CollectionRegistry, versionId: string) { + const version = config.versions.find(v => v.id === versionId) as Version | undefined + if (!version) return + + if (version.dynamic) { + await Promise.all(refs + .filter(r => localStorage.getItem(`cached_${r.id}`) !== r.hash) + .map(async r => { + await deleteMatching(url => url.startsWith(`${r.url}/${version.refs[r.id]}`)) + localStorage.setItem(`cached_${r.id}`, r.hash) + })) + } await Promise.all([ - fetchRegistries(target, version, cache, mcdataCacheValid), - fetchBlockStateMap(version, cache, mcdataCacheValid), - fetchDynamicRegistries(target, version, cache, vanillaDatapackCacheValid) + fetchRegistries(version, target), + fetchBlockStateMap(version), + fetchDynamicRegistries(version, target) ]) - - if (!mcdataCacheValid || !vanillaDatapackCacheValid) { - cache.mcdata_hash = __MCDATA_MASTER_HASH__ - cache.vanilla_datapack_summary_hash = __VANILLA_DATAPACK_SUMMARY_HASH__ - cache.format = CACHE_FORMAT - localStorage.setItem(localStorageCache(versionId), JSON.stringify(cache)) - } } -const fetchRegistries = async (target: CollectionRegistry, version: VersionConfig, cache: any, cacheValid: boolean) => { +async function fetchRegistries(version: Version, target: CollectionRegistry) { const registries = config.registries .filter(r => !r.dynamic) .filter(r => checkVersion(version.id, r.minVersion, r.maxVersion)) - if (cacheValid && cache.registries) { - registries.forEach(r => { - target.register(r.id, cache.registries[r.id]) - }) - return - } - - cache.registries = {} if (checkVersion(version.id, undefined, '1.15')) { - const url = `${mcdataUrl}/${version.mcdata_ref}/generated/reports/registries.json` + const url = `${mcdataUrl}/${version.refs.mcdata_master}/generated/reports/registries.json` try { - const res = await fetch(url) - const data = await res.json() - registries.forEach(async r => { - const values = Object.keys(data[`minecraft:${r.id}`].entries) - target.register(r.id, values) - cache.registries[r.id] = values + const data = await getData(url, (data) => { + const res: {[id: string]: string[]} = {} + Object.keys(data).forEach(k => { + res[k.slice(10)] = Object.keys(data[k].entries) + }) + return res + }) + registries.forEach(r => { + target.register(r.id, data[r.id] ?? []) }) } catch (e) { - console.warn(`Error occurred while fetching registries for version ${version.id}`) + console.warn(`Error occurred while fetching registries:`, e) } } else { - await Promise.all(registries.map(async r => { - const url = r.path - ? `${mcdataUrl}/${version.mcdata_ref}/${r.path}/data.min.json` - : `${mcdataUrl}/${version.mcdata_ref}/processed/reports/registries/${r.id}/data.min.json` - + return Promise.all(registries.map(async r => { try { - const res = await fetch(url) - const data = await res.json() - - target.register(r.id, data.values) - cache.registries[r.id] = data.values + const url = r.path + ? `${mcdataUrl}/${version.refs.mcdata_master}/${r.path}/data.min.json` + : `${mcdataUrl}/${version.refs.mcdata_master}/processed/reports/registries/${r.id}/data.min.json` + target.register(r.id, await getData(url, v => v.values)) } catch (e) { - console.warn(`Error occurred while fetching registry "${r.id}":`, e) + console.warn(`Error occurred while registry ${r.id}:`, e) } })) } } -const fetchBlockStateMap = async (version: VersionConfig, cache: any, cacheValid: boolean) => { - if (cacheValid && cache.block_state_map) { - Object.keys(cache.block_state_map).forEach(block => { - BlockStateRegistry[block] = cache.block_state_map[block] - }) - return - } - - cache.block_state_map = {} +async function fetchBlockStateMap(version: Version) { const url = (checkVersion(version.id, undefined, '1.15')) - ? `${mcdataUrl}/${version.mcdata_ref}/generated/reports/blocks.json` - : `${mcdataUrl}/${version.mcdata_ref}/processed/reports/blocks/data.min.json` + ? `${mcdataUrl}/${version.refs.mcdata_master}/generated/reports/blocks.json` + : `${mcdataUrl}/${version.refs.mcdata_master}/processed/reports/blocks/data.min.json` - const res = await fetch(url) - const data = await res.json() - - Object.keys(data).forEach(block => { - const res = { - properties: data[block].properties, - default: data[block].states.find((s: any) => s.default).properties - } - BlockStateRegistry[block] = res - cache.block_state_map[block] = res - }) + try { + const data = await getData(url, (data) => { + const res: BlockStateRegistry = {} + Object.keys(data).forEach(b => { + res[b] = { + properties: data[b].properties, + default: data[b].states.find((s: any) => s.default).properties + } + }) + return res + }) + App.blockStateRegistry = data + } catch (e) { + console.warn(`Error occurred while fetching block state map:`, e) + } } -const fetchDynamicRegistries = async (target: CollectionRegistry, version: VersionConfig, cache: any, cacheValid: boolean) => { +async function fetchDynamicRegistries(version: Version, target: CollectionRegistry) { const registries = config.registries .filter(r => r.dynamic) .filter(r => checkVersion(version.id, r.minVersion, r.maxVersion)) - if (cacheValid && cache.dynamic_registries) { - registries.forEach(r => { - target.register(r.id, cache.dynamic_registries[r.id]) - }) - return - } - - cache.dynamic_registries = {} if (checkVersion(version.id, '1.16')) { + const url = `${vanillaDatapackUrl}/${version.refs.vanilla_datapack_summary}/summary/flattened.min.json` try { - const res = await fetch(`${vanillaDatapackUrl}/${version.vanilla_datapack_summary_ref}/summary/flattened.min.json`) - const data = await res.json() + const data = await getData(url) registries.forEach(r => { target.register(r.id, data[r.id]) - cache.dynamic_registries[r.id] = data[r.id] }) } catch (e) { console.warn(`Error occurred while fetching dynamic registries:`, e) @@ -138,11 +132,37 @@ const fetchDynamicRegistries = async (target: CollectionRegistry, version: Versi } } -export const fetchPreset = async (version: VersionConfig, registry: string, id: string) => { +export async function fetchPreset(version: Version, registry: string, id: string) { try { - const res = await fetch(`${vanillaDatapackUrl}/${version.vanilla_datapack_data_ref}/data/minecraft/${registry}/${id}.json`) + const res = await fetch(`${vanillaDatapackUrl}/${version.refs.vanilla_datapack_data}/data/minecraft/${registry}/${id}.json`) return await res.json() } catch (e) { console.warn(`Error occurred while fetching ${registry} preset ${id}:`, e) } } + +async function getData(url: string, fn: (v: any) => T = (v: any) => v): Promise { + const cache = await caches.open(CACHE_NAME) + const cacheResponse = await cache.match(url) + + if (cacheResponse && cacheResponse.ok) { + return await cacheResponse.json() + } + + const fetchResponse = await fetch(url) + const responseData = fn(await fetchResponse.json()) + await cache.put(url, new Response(JSON.stringify(responseData))) + return responseData +} + +async function deleteMatching(matches: (url: string) => boolean) { + const cache = await caches.open(CACHE_NAME) + const promises: Promise[] = [] + + for (const request of await cache.keys()) { + if (matches(request.url)) { + promises.push(cache.delete(request)) + } + } + return (await Promise.all(promises)).length > 0 +} diff --git a/src/app/components/panels/TreePanel.ts b/src/app/components/panels/TreePanel.ts index e6a6b1a2..2c3c06c1 100644 --- a/src/app/components/panels/TreePanel.ts +++ b/src/app/components/panels/TreePanel.ts @@ -67,8 +67,9 @@ export const TreePanel = (view: View, model: DataModel) => { })}">${r}`).join('') } const presetControl = view.register(el => { - App.version.watchRun(v => { - const enabled = (m?.path && checkVersion(v, '1.16')) + App.schemasLoaded.watch(v => { + if (!v) return + const enabled = (m?.path && checkVersion(App.version.get(), '1.16')) el.classList.toggle('disabled', !enabled || (App.collections.get()?.get(registry) ?? []).length === 0) if (enabled) { view.mount(presetList, getPresets(), false) diff --git a/src/app/hooks/customValidation.ts b/src/app/hooks/customValidation.ts index 96009496..b55a7243 100644 --- a/src/app/hooks/customValidation.ts +++ b/src/app/hooks/customValidation.ts @@ -1,5 +1,5 @@ import { Errors, Hook, relativePath } from '@mcschema/core' -import { BlockStateRegistry } from '../App' +import { App, BlockStateRegistry } from '../App' import { getFilterKey } from './getFilterKey' import { walk } from './walk' @@ -17,7 +17,7 @@ export const customValidation: Hook<[any, Errors], void> = walk<[Errors]>({ const block = relativePath(path, config.validation.params.id).get() const errors = path.getModel().errors - const requiredProps = (BlockStateRegistry[block] ?? {}).properties ?? {} + const requiredProps = (App.blockStateRegistry[block] ?? {}).properties ?? {} const existingKeys = Object.keys(value ?? {}) Object.keys(requiredProps).forEach(p => { if (!existingKeys.includes(p)) { diff --git a/src/app/hooks/renderHtml.ts b/src/app/hooks/renderHtml.ts index 5bf360af..6feac969 100644 --- a/src/app/hooks/renderHtml.ts +++ b/src/app/hooks/renderHtml.ts @@ -113,7 +113,7 @@ export const renderHtml: Hook<[any, Mounter], [string, string, string]> = { path.model.set(path.push(key), children.default()) }) let suffix = '' - const blockState = (config.validation?.validator === 'block_state_map' ? BlockStateRegistry[relativePath(path, config.validation.params.id).get()] : null) + const blockState = (config.validation?.validator === 'block_state_map' ? App.blockStateRegistry[relativePath(path, config.validation.params.id).get()] : null) if (!blockState || blockState.properties) { const keyRendered = (blockState ? StringNode(null!, { enum: Object.keys(blockState.properties ?? {}) }) diff --git a/src/config.json b/src/config.json index 20c8061f..04901f5c 100644 --- a/src/config.json +++ b/src/config.json @@ -44,19 +44,26 @@ "versions": [ { "id": "1.15", - "mcdata_ref": "13355f7" + "refs": { + "mcdata_master": "13355f7" + } }, { "id": "1.16", - "mcdata_ref": "1.16.4", - "vanilla_datapack_data_ref": "1.16.4-data", - "vanilla_datapack_summary_ref": "1.16.4-summary" + "refs": { + "mcdata_master": "1.16.4", + "vanilla_datapack_data": "1.16.4-data", + "vanilla_datapack_summary": "1.16.4-summary" + } }, { "id": "1.17", - "mcdata_ref": "master", - "vanilla_datapack_data_ref": "data", - "vanilla_datapack_summary_ref": "summary" + "refs": { + "mcdata_master": "master", + "vanilla_datapack_data": "data", + "vanilla_datapack_summary": "summary" + }, + "dynamic": true } ], "models": [