Add indexeddb storage

2025-12-10 10:57:04 +00:00 · 2024-08-14 12:46:25 -07:00
parent d26bac8a42
commit 697e893ae6
8 changed files with 296 additions and 124 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -18,6 +18,7 @@
        "@welshman/store": "^0.0.2",
        "@welshman/util": "^0.0.25",
        "daisyui": "^4.12.10",
+        "idb": "^8.0.0",
        "nostr-tools": "^2.7.2",
        "prettier-plugin-tailwindcss": "^0.6.5",
        "throttle-debounce": "^5.0.2"
@@ -2724,6 +2725,11 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/idb": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.0.tgz",
+      "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw=="
+    },
    "node_modules/ignore": {
      "version": "5.3.1",
      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
--- a/package.json
+++ b/package.json
@@ -44,6 +44,7 @@
    "@welshman/store": "^0.0.2",
    "@welshman/util": "^0.0.25",
    "daisyui": "^4.12.10",
+    "idb": "^8.0.0",
    "nostr-tools": "^2.7.2",
    "prettier-plugin-tailwindcss": "^0.6.5",
    "throttle-debounce": "^5.0.2"
--- a/src/app/base.ts
+++ b/src/app/base.ts
@@ -1,4 +1,4 @@
-import {derived} from "svelte/store"
+import {derived, writable} from "svelte/store"
 import {memoize, assoc} from '@welshman/lib'
 import type {CustomEvent} from '@welshman/util'
 import {Repository, createEvent, Relay} from "@welshman/util"
--- a/src/app/components/LogIn.svelte
+++ b/src/app/components/LogIn.svelte
@@ -17,6 +17,7 @@
  const tryLogin = async () => {
    const secret = makeSecret()
    const handle = await loadHandle(`${username}@${handler.domain}`)
+    console.log(handle)

    if (!handle?.pubkey) {
      return pushToast({
--- a/src/app/state.ts
+++ b/src/app/state.ts
@@ -8,36 +8,46 @@ import type {SubscribeRequest} from '@welshman/net'
 import {publish, subscribe} from '@welshman/net'
 import {decrypt} from '@welshman/signer'
 import {deriveEvents, deriveEventsMapped, getter, withGetter} from "@welshman/store"
-import {synced, parseJson} from '@lib/util'
+import {parseJson} from '@lib/util'
 import type {Session, Handle, Relay} from '@app/types'
 import {INDEXER_RELAYS, DUFFLEPUD_URL, repository, pk, getSession, getSigner, signer} from "@app/base"

 // Utils

 export const createCollection = <T>({
+  name,
  store,
  getKey,
-  isStale,
  load,
 }: {
+  name: string,
  store: Readable<T[]>,
  getKey: (item: T) => string,
-  isStale: (item: T) => boolean,
  load: (key: string, ...args: any) => Promise<any>
 }) => {
  const indexStore = derived(store, $items => indexBy(getKey, $items))
  const getIndex = getter(indexStore)
-
  const getItem = (key: string) => getIndex().get(key)
+  const pending = new Map<string, Promise<Maybe<T>>>

  const loadItem = async (key: string, ...args: any[]) => {
-    const item = getIndex().get(key)
-
-    if (item && !isStale(item)) {
-      return item
+    if (getFreshness(name, key) > now() - 3600) {
+      return getIndex().get(key)
    }

-    await load(key, ...args)
+    if (pending.has(key)) {
+      await pending.get(key)
+    } else {
+      setFreshness(name, key, now())
+
+      const promise = load(key, ...args)
+
+      pending.set(key, promise)
+
+      await promise
+
+      pending.delete(key)
+    }

    return getIndex().get(key)
  }
@@ -61,9 +71,7 @@ export const load = (request: SubscribeRequest) =>
  new Promise<Maybe<CustomEvent>>(resolve => {
    const sub = subscribe({closeOnEose: true, timeout: 3000, delay: 50, ...request})

-    sub.emitter.on('event', (url: string, event: SignedEvent) => {
-      const e: CustomEvent = {...event, fetched_at: now()}
-
+    sub.emitter.on('event', (url: string, e: SignedEvent) => {
      repository.publish(e)
      sub.close()
      resolve(e)
@@ -104,6 +112,25 @@ export const updateList = async (kind: number, modifyTags: ModifyTags) => {
  await publish({event, relays})
 }

+// Freshness
+
+export const freshness = withGetter(writable<Record<string, number>>({}))
+
+export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}`
+
+export const getFreshness = (ns: string, key: string) => freshness.get()[getFreshnessKey(ns, key)] || 0
+
+export const setFreshness = (ns: string, key: string, ts: number) => freshness.update(assoc(getFreshnessKey(ns, key), ts))
+
+export const setFreshnessBulk = (ns: string, updates: Record<string, number>) =>
+  freshness.update($freshness => {
+    for (const [key, ts] of Object.entries(updates)) {
+      $freshness[key] = ts
+    }
+
+    return $freshness
+  })
+
 // Plaintext

 export const plaintext = withGetter(writable<Record<string, string>>({}))
@@ -138,23 +165,23 @@ export const {
  loadItem: loadRelay,
  // getItem: getRelay,
 } = createCollection({
-    store: relays,
-    getKey: (relay: Relay) => relay.url,
-    isStale: (relay: Relay) => relay.fetched_at < now() - 3600,
-    load: batcher(800, async (urls: string[]) => {
-      const urlSet = new Set(urls)
-      const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)})
-      const index = indexBy((item: any) => item.url, res?.data || [])
-      const items: Relay[] = urls.map(url => {
-        const {info = {}} = index.get(url) || {}
+  name: 'relays',
+  store: relays,
+  getKey: (relay: Relay) => relay.url,
+  load: batcher(800, async (urls: string[]) => {
+    const urlSet = new Set(urls)
+    const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)})
+    const index = indexBy((item: any) => item.url, res?.data || [])
+    const items: Relay[] = urls.map(url => {
+      const {info = {}} = index.get(url) || {}

-        return {...info, url, fetched_at: now()}
-      })
+      return {...info, url}
+    })

-      relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items]))
+    relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items]))

-      return items
-    }),
+    return items
+  }),
 })

 // Handles
@@ -162,29 +189,29 @@ export const {
 export const handles = writable<Handle[]>([])

 export const {
-  indexStore: handlesByPubkey,
-  getIndex: getHandlesByPubkey,
+  indexStore: handlesByNip05,
+  getIndex: getHandlesByNip05,
  deriveItem: deriveHandle,
  loadItem: loadHandle,
  // getItem: getHandle,
 } = createCollection({
-    store: handles,
-    getKey: (handle: Handle) => handle.pubkey,
-    isStale: (handle: Handle) => handle.fetched_at < now() - 3600,
-    load: batcher(800, async (nip05s: string[]) => {
-      const nip05Set = new Set(nip05s)
-      const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)})
-      const index = indexBy((item: any) => item.handle, res?.data || [])
-      const items: Handle[] = nip05s.map(nip05 => {
-        const {info = {}} = index.get(nip05) || {}
+  name: 'handles',
+  store: handles,
+  getKey: (handle: Handle) => handle.nip05,
+  load: batcher(800, async (nip05s: string[]) => {
+    const nip05Set = new Set(nip05s)
+    const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)})
+    const index = indexBy((item: any) => item.handle, res?.data || [])
+    const items: Handle[] = nip05s.map(nip05 => {
+      const {info = {}} = index.get(nip05) || {}

-        return {...info, nip05, fetched_at: now()}
-      })
+      return {...info, nip05}
+    })

-      handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items]))
+    handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items]))

-      return items
-    }),
+    return items
+  }),
 })

 // Profiles
@@ -203,15 +230,15 @@ export const {
  loadItem: loadProfile,
  // getItem: getProfile,
 } = createCollection({
-    store: profiles,
-    getKey: profile => profile.event.pubkey,
-    isStale: (profile: PublishedProfile) => profile.event.fetched_at < now() - 3600,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
-        ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [PROFILE], authors: [pubkey]}],
-      }),
+  name: 'profiles',
+  store: profiles,
+  getKey: profile => profile.event.pubkey,
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+    load({
+      ...request,
+      relays: [...relays, ...INDEXER_RELAYS],
+      filters: [{kinds: [PROFILE], authors: [pubkey]}],
+    }),
 })

 // Relay selections
@@ -231,15 +258,15 @@ export const {
  loadItem: loadRelaySelections,
  // getItem: getRelaySelections,
 } = createCollection({
-    store: relaySelections,
-    getKey: relaySelections => relaySelections.pubkey,
-    isStale: (relaySelections: CustomEvent) => relaySelections.fetched_at < now() - 3600,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
-        ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [RELAYS], authors: [pubkey]}],
-      })
+  name: 'relaySelections',
+  store: relaySelections,
+  getKey: relaySelections => relaySelections.pubkey,
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+    load({
+      ...request,
+      relays: [...relays, ...INDEXER_RELAYS],
+      filters: [{kinds: [RELAYS], authors: [pubkey]}],
+    })
 })

 // Follows
@@ -263,15 +290,15 @@ export const {
  loadItem: loadFollows,
  // getItem: getFollows,
 } = createCollection({
-    store: follows,
-    getKey: follows => follows.event.pubkey,
-    isStale: (follows: PublishedList) => follows.event.fetched_at < now() - 3600,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
-        ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [FOLLOWS], authors: [pubkey]}],
-      })
+  name: 'follows',
+  store: follows,
+  getKey: follows => follows.event.pubkey,
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+    load({
+      ...request,
+      relays: [...relays, ...INDEXER_RELAYS],
+      filters: [{kinds: [FOLLOWS], authors: [pubkey]}],
+    })
 })

 // Mutes
@@ -295,15 +322,15 @@ export const {
  loadItem: loadMutes,
  // getItem: getMutes,
 } = createCollection({
-    store: mutes,
-    getKey: mute => mute.event.pubkey,
-    isStale: (mutes: PublishedList) => mutes.event.fetched_at < now() - 3600,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
-        ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [MUTES], authors: [pubkey]}],
-      })
+  name: 'mutes',
+  store: mutes,
+  getKey: mute => mute.event.pubkey,
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+    load({
+      ...request,
+      relays: [...relays, ...INDEXER_RELAYS],
+      filters: [{kinds: [MUTES], authors: [pubkey]}],
+    })
 })

 // Groups
@@ -360,18 +387,18 @@ export const {
  loadItem: loadGroup,
  // getItem: getGroup,
 } = createCollection({
-    store: groups,
-    getKey: (group: PublishedGroup) => group.nom,
-    isStale: (group: PublishedGroup) => group.event.fetched_at < now() - 3600,
-    load: (nom: string, relays: string[] = [], request: Partial<SubscribeRequest> = {}) =>
-      Promise.all([
-        ...relays.map(loadRelay),
-        load({
-          ...request,
-          relays,
-          filters: [{kinds: [GROUP_META], '#d': [nom]}],
-        }),
-      ])
+  name: 'groups',
+  store: groups,
+  getKey: (group: PublishedGroup) => group.nom,
+  load: (nom: string, relays: string[] = [], request: Partial<SubscribeRequest> = {}) =>
+    Promise.all([
+      ...relays.map(loadRelay),
+      load({
+        ...request,
+        relays,
+        filters: [{kinds: [GROUP_META], '#d': [nom]}],
+      }),
+    ])
 })

 // Qualified groups
@@ -433,9 +460,9 @@ export const {
  loadItem: loadGroupMembership,
  // getItem: getGroupMembership,
 } = createCollection({
+    name: 'groupMemberships',
    store: groupMemberships,
    getKey: groupMembership => groupMembership.event.pubkey,
-    isStale: (groupMembership: PublishedGroupMembership) => groupMembership.event.fetched_at < now() - 3600,
    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
      load({
        ...request,
--- a/src/app/storage.ts
+++ b/src/app/storage.ts
@@ -0,0 +1,123 @@
+import {openDB, deleteDB} from "idb"
+import type {IDBPDatabase} from "idb"
+import {throttle} from 'throttle-debounce'
+import {writable} from 'svelte/store'
+import type {Unsubscriber, Writable} from 'svelte/store'
+import {isNil, randomInt} from '@welshman/lib'
+import {withGetter} from '@welshman/store'
+import {getJson, setJson} from '@lib/util'
+import {pk, sessions, repository} from '@app/base'
+
+export type Item = Record<string, any>
+
+export type IndexedDbAdapter = {
+  keyPath: string
+  store: Writable<Item[]>
+}
+
+export let db: IDBPDatabase
+
+export const dead = withGetter(writable(false))
+
+export const subs: Unsubscriber[] = []
+
+export const DB_NAME = "flotilla"
+
+export const DB_VERSION = 1
+
+export const getAll = async (name: string) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+  const result = await store.getAll()
+
+  await tx.done
+
+  return result
+}
+
+export const bulkPut = async (name: string, data: any[]) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+
+  await Promise.all(data.map(item => store.put(item)))
+  await tx.done
+}
+
+export const bulkDelete = async (name: string, ids: string[]) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+
+  await Promise.all(ids.map(id => store.delete(id)))
+  await tx.done
+}
+
+export const initIndexedDbAdapter = async (name: string, adapter: IndexedDbAdapter) => {
+  let copy = await getAll(name)
+
+  adapter.store.set(copy)
+
+  adapter.store.subscribe(
+    throttle(randomInt(3000, 5000), async (data: Item[]) => {
+      if (dead.get()) {
+        return
+      }
+
+      const prevIds = new Set(copy.map(item => item[adapter.keyPath]))
+      const currentIds = new Set(data.map(item => item[adapter.keyPath]))
+      const newRecords = data.filter(r => !prevIds.has(r[adapter.keyPath]))
+      const removedRecords = copy.filter(r => !currentIds.has(r[adapter.keyPath]))
+
+      copy = data
+
+      if (newRecords.length > 0) {
+        await bulkPut(name, newRecords)
+      }
+
+      if (removedRecords.length > 0) {
+        await bulkDelete(name, removedRecords.map(item => item[adapter.keyPath]))
+      }
+    }),
+  )
+}
+
+export const initStorage = async (adapters: Record<string, IndexedDbAdapter>) => {
+  if (!window.indexedDB) return
+
+  window.addEventListener("beforeunload", () => closeStorage())
+
+  db = await openDB(DB_NAME, DB_VERSION, {
+    upgrade(db: IDBPDatabase) {
+      const names = Object.keys(adapters)
+
+      for (const name of db.objectStoreNames) {
+        if (!names.includes(name)) {
+          db.deleteObjectStore(name)
+        }
+      }
+
+      for (const [name, {keyPath}] of Object.entries(adapters)) {
+        try {
+          db.createObjectStore(name, {keyPath})
+        } catch (e) {
+          console.warn(e)
+        }
+      }
+    },
+  })
+
+  await Promise.all(
+    Object.entries(adapters)
+      .map(([name, config]) => initIndexedDbAdapter(name, config))
+  )
+}
+
+export const closeStorage = async () => {
+  dead.set(true)
+  subs.forEach(unsub => unsub())
+  await db?.close()
+}
+
+export const clearStorage = async () => {
+  await closeStorage()
+  await deleteDB(DB_NAME)
+}
--- a/src/app/types.ts
+++ b/src/app/types.ts
@@ -9,20 +9,11 @@ export type Session = {
  handler?: Nip46Handler
 }

-export type Relay = RelayProfile & {
-  fetched_at: number,
-}
+export type Relay = RelayProfile

 export type Handle = {
  pubkey: string
  nip05: string
  nip46: string[]
  relays: string[]
-  fetched_at: number,
-}
-
-declare module '@welshman/util' {
-  interface CustomEvent extends TrustedEvent {
-    fetched_at: number
-  }
 }
--- a/src/routes/+layout.svelte
+++ b/src/routes/+layout.svelte
@@ -2,6 +2,7 @@
  import "@src/app.css"
  import {onMount} from 'svelte'
  import {page} from "$app/stores"
+  import {createEventStore} from '@welshman/store'
  import {fly} from "@lib/transition"
  import ModalBox from "@lib/components/ModalBox.svelte"
  import Toast from "@app/components/Toast.svelte"
@@ -9,8 +10,11 @@
  import PrimaryNav from "@app/components/PrimaryNav.svelte"
  import SecondaryNav from "@app/components/SecondaryNav.svelte"
  import {modals, clearModal} from "@app/modal"
-  import {session} from "@app/base"
+  import {session, sessions, pk, repository} from "@app/base"
+  import {plaintext, relays, handles} from "@app/state"
+  import {initStorage} from "@app/storage"

+  let ready: Promise<void>
  let dialog: HTMLDialogElement
  let prev: any

@@ -31,6 +35,21 @@
  }

  onMount(() => {
+    ready = initStorage({
+      events: {
+        keyPath: 'id',
+        store: createEventStore(repository),
+      },
+      relays: {
+        keyPath: 'url',
+        store: relays,
+      },
+      handles: {
+        keyPath: 'nip05',
+        store: handles,
+      },
+    })
+
    dialog.addEventListener('close', () => {
      if (modal) {
        clearModal()
@@ -39,26 +58,30 @@
  })
 </script>

-<div data-theme="dark">
-  <div class="flex h-screen">
-    <PrimaryNav />
-    <SecondaryNav />
-    <div class="flex-grow bg-base-200">
-      <slot />
+{#await ready}
+  <div data-theme="dark" />
+{:then}
+  <div data-theme="dark">
+    <div class="flex h-screen">
+      <PrimaryNav />
+      <SecondaryNav />
+      <div class="flex-grow bg-base-200">
+        <slot />
+      </div>
    </div>
+    <dialog bind:this={dialog} class="modal modal-bottom sm:modal-middle !z-modal">
+      {#if prev}
+        {#key prev}
+          <ModalBox {...prev} />
+        {/key}
+        <Toast />
+      {/if}
+      {#if $session}
+        <form method="dialog" class="modal-backdrop">
+          <button />
+        </form>
+      {/if}
+    </dialog>
+    <Toast />
  </div>
-  <dialog bind:this={dialog} class="modal modal-bottom sm:modal-middle !z-modal">
-    {#if prev}
-      {#key prev}
-        <ModalBox {...prev} />
-      {/key}
-      <Toast />
-    {/if}
-    {#if $session}
-      <form method="dialog" class="modal-backdrop">
-        <button />
-      </form>
-    {/if}
-  </dialog>
-  <Toast />
-</div>
+{/await}