Use Cache API instead of LocalStorage (#128)

* Use Cache API instead of LocalStorage

* Remove old cached data

src/app/App.ts

@@ -40,7 +40,7 @@ export const Models: {
 config.models.filter(m => m.schema)
   .forEach(m => Models[m.id] = new DataModel(ObjectNode({})))
 
-export let BlockStateRegistry: {
+export type BlockStateRegistry = {
   [block: string]: {
     properties: {
       [key: string]: string[]
@@ -49,7 +49,7 @@ export let BlockStateRegistry: {
       [key: string]: string
     }
   }
-} = {}
+}
 
 export const App = {
   version: new LocalStorageProperty('schema_version', config.versions[config.versions.length - 1].id)
@@ -70,7 +70,8 @@ export const App = {
   localesLoaded: new Property(false),
   loaded: new Property(false),
   mobilePanel: new Property('tree'),
-  settings: new Settings('generator_settings')
+  settings: new Settings('generator_settings'),
+  blockStateRegistry: {} as BlockStateRegistry
 }
 
 App.version.watchRun(async (value) => {
@@ -108,7 +109,7 @@ App.mobilePanel.watchRun((value) => {
 })
 
 async function updateSchemas(version: string) {
-  BlockStateRegistry = {}
+  App.blockStateRegistry = {}
   const collections = Versions[version].getCollections()
   App.collections.set(collections)
   await fetchData(collections, version)

src/app/DataFetcher.ts

@@ -1,136 +1,130 @@
 import { CollectionRegistry } from '@mcschema/core'
-import { BlockStateRegistry, checkVersion } from './App'
+import { App, BlockStateRegistry, checkVersion } from './App'
 import config from '../config.json'
 
-const CACHE_FORMAT = 1
+['1.15', '1.16', '1.17'].forEach(v => localStorage.removeItem(`cache_${v}`))
 
-type VersionConfig = {
+const CACHE_NAME = `misode-v1`
+
+type VersionRef = 'mcdata_master' | 'vanilla_datapack_summary' | 'vanilla_datapack_data'
+
+type Version = {
   id: string,
-  mcdata_ref: string,
-  vanilla_datapack_data_ref?: string,
-  vanilla_datapack_summary_ref?: string
+  refs: Partial<{ [key in VersionRef]: string }>,
+  dynamic?: boolean,
 }
 
-const localStorageCache = (version: string) => `cache_${version}`
 declare var __MCDATA_MASTER_HASH__: string;
 declare var __VANILLA_DATAPACK_SUMMARY_HASH__: string;
 
 const mcdataUrl = 'https://raw.githubusercontent.com/Arcensoth/mcdata'
 const vanillaDatapackUrl = 'https://raw.githubusercontent.com/SPGoding/vanilla-datapack'
 
-export const fetchData = async (target: CollectionRegistry, versionId: string) => {
-  const version = config.versions.find(v => v.id === versionId)
-  if (!version) return
+const refs: {
+  id: VersionRef,
+  hash: string,
+  url: string
+}[] = [
+  {
+    id: 'mcdata_master',
+    hash: __MCDATA_MASTER_HASH__,
+    url: mcdataUrl
+  },
+  {
+    id: 'vanilla_datapack_summary',
+    hash: __VANILLA_DATAPACK_SUMMARY_HASH__,
+    url: vanillaDatapackUrl
+  },
+]
 
-  const cache = JSON.parse(localStorage.getItem(localStorageCache(versionId)) ?? '{}')
-  const mcdataCacheValid = cache.format === CACHE_FORMAT && (version.mcdata_ref !== 'master' || cache.mcdata_hash === __MCDATA_MASTER_HASH__)
-  const vanillaDatapackCacheValid = cache.format === CACHE_FORMAT && (version.vanilla_datapack_summary_ref !== 'summary' || cache.vanilla_datapack_summary_hash === __VANILLA_DATAPACK_SUMMARY_HASH__)
+export async function fetchData(target: CollectionRegistry, versionId: string) {
+  const version = config.versions.find(v => v.id === versionId) as Version | undefined
+  if (!version) return
+  
+  if (version.dynamic) {
+    await Promise.all(refs
+      .filter(r => localStorage.getItem(`cached_${r.id}`) !== r.hash)
+      .map(async r => {
+        await deleteMatching(url => url.startsWith(`${r.url}/${version.refs[r.id]}`))
+        localStorage.setItem(`cached_${r.id}`, r.hash)
+      }))
+  }
 
   await Promise.all([
-    fetchRegistries(target, version, cache, mcdataCacheValid),
-    fetchBlockStateMap(version, cache, mcdataCacheValid),
-    fetchDynamicRegistries(target, version, cache, vanillaDatapackCacheValid)
+    fetchRegistries(version, target),
+    fetchBlockStateMap(version),
+    fetchDynamicRegistries(version, target)
   ])
-
-  if (!mcdataCacheValid || !vanillaDatapackCacheValid) {
-    cache.mcdata_hash = __MCDATA_MASTER_HASH__
-    cache.vanilla_datapack_summary_hash = __VANILLA_DATAPACK_SUMMARY_HASH__
-    cache.format = CACHE_FORMAT
-    localStorage.setItem(localStorageCache(versionId), JSON.stringify(cache))
-  }
 }
 
-const fetchRegistries = async (target: CollectionRegistry, version: VersionConfig, cache: any, cacheValid: boolean) => {
+async function fetchRegistries(version: Version, target: CollectionRegistry) {
   const registries = config.registries
     .filter(r => !r.dynamic)
     .filter(r => checkVersion(version.id, r.minVersion, r.maxVersion))
 
-  if (cacheValid && cache.registries) {
-    registries.forEach(r => {
-      target.register(r.id, cache.registries[r.id])
-    })
-    return
-  }
-
-  cache.registries = {}
   if (checkVersion(version.id, undefined, '1.15')) {
-    const url = `${mcdataUrl}/${version.mcdata_ref}/generated/reports/registries.json`
+    const url = `${mcdataUrl}/${version.refs.mcdata_master}/generated/reports/registries.json`
     try {
-      const res = await fetch(url)
-      const data = await res.json()
-      registries.forEach(async r => {
-        const values = Object.keys(data[`minecraft:${r.id}`].entries)
-        target.register(r.id, values)
-        cache.registries[r.id] = values
+      const data = await getData(url, (data) => {
+        const res: {[id: string]: string[]} = {}
+        Object.keys(data).forEach(k => {
+          res[k.slice(10)] = Object.keys(data[k].entries)
+        })
+        return res
+      })
+      registries.forEach(r => {
+        target.register(r.id, data[r.id] ?? [])
       })
     } catch (e) {
-      console.warn(`Error occurred while fetching registries for version ${version.id}`)
+      console.warn(`Error occurred while fetching registries:`, e)
     }
   } else {
-    await Promise.all(registries.map(async r => {
-      const url = r.path
-        ? `${mcdataUrl}/${version.mcdata_ref}/${r.path}/data.min.json`
-        : `${mcdataUrl}/${version.mcdata_ref}/processed/reports/registries/${r.id}/data.min.json`
-  
+    return Promise.all(registries.map(async r => {
       try {
-        const res = await fetch(url)
-        const data = await res.json()
-  
-        target.register(r.id, data.values)
-        cache.registries[r.id] = data.values
+        const url = r.path
+          ? `${mcdataUrl}/${version.refs.mcdata_master}/${r.path}/data.min.json`
+          : `${mcdataUrl}/${version.refs.mcdata_master}/processed/reports/registries/${r.id}/data.min.json`
+        target.register(r.id, await getData(url, v => v.values))
       } catch (e) {
-        console.warn(`Error occurred while fetching registry "${r.id}":`, e)
+        console.warn(`Error occurred while registry ${r.id}:`, e)
       }
     }))
   }
 }
 
-const fetchBlockStateMap = async (version: VersionConfig, cache: any, cacheValid: boolean) => {
-  if (cacheValid && cache.block_state_map) {
-    Object.keys(cache.block_state_map).forEach(block => {
-      BlockStateRegistry[block] = cache.block_state_map[block]
-    })
-    return
-  }
-
-  cache.block_state_map = {}
+async function fetchBlockStateMap(version: Version) {
   const url = (checkVersion(version.id, undefined, '1.15'))
-    ? `${mcdataUrl}/${version.mcdata_ref}/generated/reports/blocks.json`
-    : `${mcdataUrl}/${version.mcdata_ref}/processed/reports/blocks/data.min.json`
+    ? `${mcdataUrl}/${version.refs.mcdata_master}/generated/reports/blocks.json`
+    : `${mcdataUrl}/${version.refs.mcdata_master}/processed/reports/blocks/data.min.json`
 
-  const res = await fetch(url)
-  const data = await res.json()
-
-  Object.keys(data).forEach(block => {
-    const res = {
-      properties: data[block].properties,
-      default: data[block].states.find((s: any) => s.default).properties
-    }
-    BlockStateRegistry[block] = res
-    cache.block_state_map[block] = res
-  })
+  try {
+    const data = await getData(url, (data) => {
+      const res: BlockStateRegistry = {}
+      Object.keys(data).forEach(b => {
+        res[b] = {
+          properties: data[b].properties,
+          default: data[b].states.find((s: any) => s.default).properties
+        }
+      })
+      return res
+    })
+    App.blockStateRegistry = data
+  } catch (e) {
+    console.warn(`Error occurred while fetching block state map:`, e)
+  }
 }
 
-const fetchDynamicRegistries = async (target: CollectionRegistry, version: VersionConfig, cache: any, cacheValid: boolean) => {
+async function fetchDynamicRegistries(version: Version, target: CollectionRegistry) {
   const registries = config.registries
     .filter(r => r.dynamic)
     .filter(r => checkVersion(version.id, r.minVersion, r.maxVersion))
 
-  if (cacheValid && cache.dynamic_registries) {
-    registries.forEach(r => {
-      target.register(r.id, cache.dynamic_registries[r.id])
-    })
-    return
-  }
-
-  cache.dynamic_registries = {}
   if (checkVersion(version.id, '1.16')) {
+    const url = `${vanillaDatapackUrl}/${version.refs.vanilla_datapack_summary}/summary/flattened.min.json`
     try {
-      const res = await fetch(`${vanillaDatapackUrl}/${version.vanilla_datapack_summary_ref}/summary/flattened.min.json`)
-      const data = await res.json()
+      const data = await getData(url)
       registries.forEach(r => {
         target.register(r.id, data[r.id])
-        cache.dynamic_registries[r.id] = data[r.id]
       })
     } catch (e) {
       console.warn(`Error occurred while fetching dynamic registries:`, e)
@@ -138,11 +132,37 @@ const fetchDynamicRegistries = async (target: CollectionRegistry, version: Versi
   }
 }
 
-export const fetchPreset = async (version: VersionConfig, registry: string, id: string) => {
+export async function fetchPreset(version: Version, registry: string, id: string) {
   try {
-    const res = await fetch(`${vanillaDatapackUrl}/${version.vanilla_datapack_data_ref}/data/minecraft/${registry}/${id}.json`)
+    const res = await fetch(`${vanillaDatapackUrl}/${version.refs.vanilla_datapack_data}/data/minecraft/${registry}/${id}.json`)
     return await res.json()
   } catch (e) {
     console.warn(`Error occurred while fetching ${registry} preset ${id}:`, e)
   }
 }
+
+async function getData<T = any>(url: string, fn: (v: any) => T = (v: any) => v): Promise<T> {
+  const cache = await caches.open(CACHE_NAME)
+  const cacheResponse = await cache.match(url)
+
+  if (cacheResponse && cacheResponse.ok) {
+    return await cacheResponse.json()
+  }
+
+  const fetchResponse = await fetch(url)
+  const responseData = fn(await fetchResponse.json())
+  await cache.put(url, new Response(JSON.stringify(responseData)))
+  return responseData
+}
+
+async function deleteMatching(matches: (url: string) => boolean) {
+  const cache = await caches.open(CACHE_NAME)
+  const promises: Promise<boolean>[] = []
+
+  for (const request of await cache.keys()) {
+    if (matches(request.url)) {
+      promises.push(cache.delete(request))
+    }
+  }
+  return (await Promise.all(promises)).length > 0
+}

src/app/components/panels/TreePanel.ts

@@ -67,8 +67,9 @@ export const TreePanel = (view: View, model: DataModel) => {
     })}">${r}</div>`).join('')
   }
   const presetControl = view.register(el => {
-    App.version.watchRun(v => {
-      const enabled = (m?.path && checkVersion(v, '1.16'))
+    App.schemasLoaded.watch(v => {
+      if (!v) return
+      const enabled = (m?.path && checkVersion(App.version.get(), '1.16'))
       el.classList.toggle('disabled', !enabled || (App.collections.get()?.get(registry) ?? []).length === 0)
       if (enabled) {
         view.mount(presetList, getPresets(), false)

src/app/hooks/customValidation.ts

@@ -1,5 +1,5 @@
 import { Errors, Hook, relativePath } from '@mcschema/core'
-import { BlockStateRegistry } from '../App'
+import { App, BlockStateRegistry } from '../App'
 import { getFilterKey } from './getFilterKey'
 import { walk } from './walk'
 
@@ -17,7 +17,7 @@ export const customValidation: Hook<[any, Errors], void> = walk<[Errors]>({
       const block = relativePath(path, config.validation.params.id).get()
       const errors = path.getModel().errors
 
-      const requiredProps = (BlockStateRegistry[block] ?? {}).properties ?? {}
+      const requiredProps = (App.blockStateRegistry[block] ?? {}).properties ?? {}
       const existingKeys = Object.keys(value ?? {})
       Object.keys(requiredProps).forEach(p => {
         if (!existingKeys.includes(p)) {

src/app/hooks/renderHtml.ts

@@ -113,7 +113,7 @@ export const renderHtml: Hook<[any, Mounter], [string, string, string]> = {
       path.model.set(path.push(key), children.default())
     })
     let suffix = ''
-    const blockState = (config.validation?.validator === 'block_state_map' ? BlockStateRegistry[relativePath(path, config.validation.params.id).get()] : null)
+    const blockState = (config.validation?.validator === 'block_state_map' ? App.blockStateRegistry[relativePath(path, config.validation.params.id).get()] : null)
     if (!blockState || blockState.properties) {
       const keyRendered = (blockState
         ? StringNode(null!, { enum: Object.keys(blockState.properties ?? {}) })