Add indexeddb storage

package-lock.json

@@ -18,6 +18,7 @@
         "@welshman/store": "^0.0.2",
         "@welshman/util": "^0.0.25",
         "daisyui": "^4.12.10",
+        "idb": "^8.0.0",
         "nostr-tools": "^2.7.2",
         "prettier-plugin-tailwindcss": "^0.6.5",
         "throttle-debounce": "^5.0.2"
@@ -2724,6 +2725,11 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/idb": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.0.tgz",
+      "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw=="
+    },
     "node_modules/ignore": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",

package.json

@@ -44,6 +44,7 @@
     "@welshman/store": "^0.0.2",
     "@welshman/util": "^0.0.25",
     "daisyui": "^4.12.10",
+    "idb": "^8.0.0",
     "nostr-tools": "^2.7.2",
     "prettier-plugin-tailwindcss": "^0.6.5",
     "throttle-debounce": "^5.0.2"

src/app/base.ts

@@ -1,4 +1,4 @@
-import {derived} from "svelte/store"
+import {derived, writable} from "svelte/store"
 import {memoize, assoc} from '@welshman/lib'
 import type {CustomEvent} from '@welshman/util'
 import {Repository, createEvent, Relay} from "@welshman/util"

src/app/components/LogIn.svelte

@@ -17,6 +17,7 @@
   const tryLogin = async () => {
     const secret = makeSecret()
     const handle = await loadHandle(`${username}@${handler.domain}`)
+    console.log(handle)
 
     if (!handle?.pubkey) {
       return pushToast({

src/app/state.ts

@@ -8,36 +8,46 @@ import type {SubscribeRequest} from '@welshman/net'
 import {publish, subscribe} from '@welshman/net'
 import {decrypt} from '@welshman/signer'
 import {deriveEvents, deriveEventsMapped, getter, withGetter} from "@welshman/store"
-import {synced, parseJson} from '@lib/util'
+import {parseJson} from '@lib/util'
 import type {Session, Handle, Relay} from '@app/types'
 import {INDEXER_RELAYS, DUFFLEPUD_URL, repository, pk, getSession, getSigner, signer} from "@app/base"
 
 // Utils
 
 export const createCollection = <T>({
+  name,
   store,
   getKey,
-  isStale,
   load,
 }: {
+  name: string,
   store: Readable<T[]>,
   getKey: (item: T) => string,
-  isStale: (item: T) => boolean,
   load: (key: string, ...args: any) => Promise<any>
 }) => {
   const indexStore = derived(store, $items => indexBy(getKey, $items))
   const getIndex = getter(indexStore)
-
   const getItem = (key: string) => getIndex().get(key)
+  const pending = new Map<string, Promise<Maybe<T>>>
 
   const loadItem = async (key: string, ...args: any[]) => {
-    const item = getIndex().get(key)
+    if (getFreshness(name, key) > now() - 3600) {
-
+      return getIndex().get(key)
-    if (item && !isStale(item)) {
-      return item
     }
 
-    await load(key, ...args)
+    if (pending.has(key)) {
+      await pending.get(key)
+    } else {
+      setFreshness(name, key, now())
+
+      const promise = load(key, ...args)
+
+      pending.set(key, promise)
+
+      await promise
+
+      pending.delete(key)
+    }
 
     return getIndex().get(key)
   }
@@ -61,9 +71,7 @@ export const load = (request: SubscribeRequest) =>
   new Promise<Maybe<CustomEvent>>(resolve => {
     const sub = subscribe({closeOnEose: true, timeout: 3000, delay: 50, ...request})
 
-    sub.emitter.on('event', (url: string, event: SignedEvent) => {
+    sub.emitter.on('event', (url: string, e: SignedEvent) => {
-      const e: CustomEvent = {...event, fetched_at: now()}
-
       repository.publish(e)
       sub.close()
       resolve(e)
@@ -104,6 +112,25 @@ export const updateList = async (kind: number, modifyTags: ModifyTags) => {
   await publish({event, relays})
 }
 
+// Freshness
+
+export const freshness = withGetter(writable<Record<string, number>>({}))
+
+export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}`
+
+export const getFreshness = (ns: string, key: string) => freshness.get()[getFreshnessKey(ns, key)] || 0
+
+export const setFreshness = (ns: string, key: string, ts: number) => freshness.update(assoc(getFreshnessKey(ns, key), ts))
+
+export const setFreshnessBulk = (ns: string, updates: Record<string, number>) =>
+  freshness.update($freshness => {
+    for (const [key, ts] of Object.entries(updates)) {
+      $freshness[key] = ts
+    }
+
+    return $freshness
+  })
+
 // Plaintext
 
 export const plaintext = withGetter(writable<Record<string, string>>({}))
@@ -138,23 +165,23 @@ export const {
   loadItem: loadRelay,
   // getItem: getRelay,
 } = createCollection({
-    store: relays,
+  name: 'relays',
-    getKey: (relay: Relay) => relay.url,
+  store: relays,
-    isStale: (relay: Relay) => relay.fetched_at < now() - 3600,
+  getKey: (relay: Relay) => relay.url,
-    load: batcher(800, async (urls: string[]) => {
+  load: batcher(800, async (urls: string[]) => {
-      const urlSet = new Set(urls)
+    const urlSet = new Set(urls)
-      const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)})
+    const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)})
-      const index = indexBy((item: any) => item.url, res?.data || [])
+    const index = indexBy((item: any) => item.url, res?.data || [])
-      const items: Relay[] = urls.map(url => {
+    const items: Relay[] = urls.map(url => {
-        const {info = {}} = index.get(url) || {}
+      const {info = {}} = index.get(url) || {}
 
-        return {...info, url, fetched_at: now()}
+      return {...info, url}
-      })
+    })
 
-      relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items]))
+    relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items]))
 
-      return items
+    return items
-    }),
+  }),
 })
 
 // Handles
@@ -162,29 +189,29 @@ export const {
 export const handles = writable<Handle[]>([])
 
 export const {
-  indexStore: handlesByPubkey,
+  indexStore: handlesByNip05,
-  getIndex: getHandlesByPubkey,
+  getIndex: getHandlesByNip05,
   deriveItem: deriveHandle,
   loadItem: loadHandle,
   // getItem: getHandle,
 } = createCollection({
-    store: handles,
+  name: 'handles',
-    getKey: (handle: Handle) => handle.pubkey,
+  store: handles,
-    isStale: (handle: Handle) => handle.fetched_at < now() - 3600,
+  getKey: (handle: Handle) => handle.nip05,
-    load: batcher(800, async (nip05s: string[]) => {
+  load: batcher(800, async (nip05s: string[]) => {
-      const nip05Set = new Set(nip05s)
+    const nip05Set = new Set(nip05s)
-      const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)})
+    const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)})
-      const index = indexBy((item: any) => item.handle, res?.data || [])
+    const index = indexBy((item: any) => item.handle, res?.data || [])
-      const items: Handle[] = nip05s.map(nip05 => {
+    const items: Handle[] = nip05s.map(nip05 => {
-        const {info = {}} = index.get(nip05) || {}
+      const {info = {}} = index.get(nip05) || {}
 
-        return {...info, nip05, fetched_at: now()}
+      return {...info, nip05}
-      })
+    })
 
-      handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items]))
+    handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items]))
 
-      return items
+    return items
-    }),
+  }),
 })
 
 // Profiles
@@ -203,15 +230,15 @@ export const {
   loadItem: loadProfile,
   // getItem: getProfile,
 } = createCollection({
-    store: profiles,
+  name: 'profiles',
-    getKey: profile => profile.event.pubkey,
+  store: profiles,
-    isStale: (profile: PublishedProfile) => profile.event.fetched_at < now() - 3600,
+  getKey: profile => profile.event.pubkey,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
+    load({
-        ...request,
+      ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
+      relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [PROFILE], authors: [pubkey]}],
+      filters: [{kinds: [PROFILE], authors: [pubkey]}],
-      }),
+    }),
 })
 
 // Relay selections
@@ -231,15 +258,15 @@ export const {
   loadItem: loadRelaySelections,
   // getItem: getRelaySelections,
 } = createCollection({
-    store: relaySelections,
+  name: 'relaySelections',
-    getKey: relaySelections => relaySelections.pubkey,
+  store: relaySelections,
-    isStale: (relaySelections: CustomEvent) => relaySelections.fetched_at < now() - 3600,
+  getKey: relaySelections => relaySelections.pubkey,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
+    load({
-        ...request,
+      ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
+      relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [RELAYS], authors: [pubkey]}],
+      filters: [{kinds: [RELAYS], authors: [pubkey]}],
-      })
+    })
 })
 
 // Follows
@@ -263,15 +290,15 @@ export const {
   loadItem: loadFollows,
   // getItem: getFollows,
 } = createCollection({
-    store: follows,
+  name: 'follows',
-    getKey: follows => follows.event.pubkey,
+  store: follows,
-    isStale: (follows: PublishedList) => follows.event.fetched_at < now() - 3600,
+  getKey: follows => follows.event.pubkey,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
+    load({
-        ...request,
+      ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
+      relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [FOLLOWS], authors: [pubkey]}],
+      filters: [{kinds: [FOLLOWS], authors: [pubkey]}],
-      })
+    })
 })
 
 // Mutes
@@ -295,15 +322,15 @@ export const {
   loadItem: loadMutes,
   // getItem: getMutes,
 } = createCollection({
-    store: mutes,
+  name: 'mutes',
-    getKey: mute => mute.event.pubkey,
+  store: mutes,
-    isStale: (mutes: PublishedList) => mutes.event.fetched_at < now() - 3600,
+  getKey: mute => mute.event.pubkey,
-    load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
+  load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
-      load({
+    load({
-        ...request,
+      ...request,
-        relays: [...relays, ...INDEXER_RELAYS],
+      relays: [...relays, ...INDEXER_RELAYS],
-        filters: [{kinds: [MUTES], authors: [pubkey]}],
+      filters: [{kinds: [MUTES], authors: [pubkey]}],
-      })
+    })
 })
 
 // Groups
@@ -360,18 +387,18 @@ export const {
   loadItem: loadGroup,
   // getItem: getGroup,
 } = createCollection({
-    store: groups,
+  name: 'groups',
-    getKey: (group: PublishedGroup) => group.nom,
+  store: groups,
-    isStale: (group: PublishedGroup) => group.event.fetched_at < now() - 3600,
+  getKey: (group: PublishedGroup) => group.nom,
-    load: (nom: string, relays: string[] = [], request: Partial<SubscribeRequest> = {}) =>
+  load: (nom: string, relays: string[] = [], request: Partial<SubscribeRequest> = {}) =>
-      Promise.all([
+    Promise.all([
-        ...relays.map(loadRelay),
+      ...relays.map(loadRelay),
-        load({
+      load({
-          ...request,
+        ...request,
-          relays,
+        relays,
-          filters: [{kinds: [GROUP_META], '#d': [nom]}],
+        filters: [{kinds: [GROUP_META], '#d': [nom]}],
-        }),
+      }),
-      ])
+    ])
 })
 
 // Qualified groups
@@ -433,9 +460,9 @@ export const {
   loadItem: loadGroupMembership,
   // getItem: getGroupMembership,
 } = createCollection({
+    name: 'groupMemberships',
     store: groupMemberships,
     getKey: groupMembership => groupMembership.event.pubkey,
-    isStale: (groupMembership: PublishedGroupMembership) => groupMembership.event.fetched_at < now() - 3600,
     load: (pubkey: string, relays = [], request: Partial<SubscribeRequest> = {}) =>
       load({
         ...request,

src/app/storage.ts

@@ -0,0 +1,123 @@
+import {openDB, deleteDB} from "idb"
+import type {IDBPDatabase} from "idb"
+import {throttle} from 'throttle-debounce'
+import {writable} from 'svelte/store'
+import type {Unsubscriber, Writable} from 'svelte/store'
+import {isNil, randomInt} from '@welshman/lib'
+import {withGetter} from '@welshman/store'
+import {getJson, setJson} from '@lib/util'
+import {pk, sessions, repository} from '@app/base'
+
+export type Item = Record<string, any>
+
+export type IndexedDbAdapter = {
+  keyPath: string
+  store: Writable<Item[]>
+}
+
+export let db: IDBPDatabase
+
+export const dead = withGetter(writable(false))
+
+export const subs: Unsubscriber[] = []
+
+export const DB_NAME = "flotilla"
+
+export const DB_VERSION = 1
+
+export const getAll = async (name: string) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+  const result = await store.getAll()
+
+  await tx.done
+
+  return result
+}
+
+export const bulkPut = async (name: string, data: any[]) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+
+  await Promise.all(data.map(item => store.put(item)))
+  await tx.done
+}
+
+export const bulkDelete = async (name: string, ids: string[]) => {
+  const tx = db.transaction(name, "readwrite")
+  const store = tx.objectStore(name)
+
+  await Promise.all(ids.map(id => store.delete(id)))
+  await tx.done
+}
+
+export const initIndexedDbAdapter = async (name: string, adapter: IndexedDbAdapter) => {
+  let copy = await getAll(name)
+
+  adapter.store.set(copy)
+
+  adapter.store.subscribe(
+    throttle(randomInt(3000, 5000), async (data: Item[]) => {
+      if (dead.get()) {
+        return
+      }
+
+      const prevIds = new Set(copy.map(item => item[adapter.keyPath]))
+      const currentIds = new Set(data.map(item => item[adapter.keyPath]))
+      const newRecords = data.filter(r => !prevIds.has(r[adapter.keyPath]))
+      const removedRecords = copy.filter(r => !currentIds.has(r[adapter.keyPath]))
+
+      copy = data
+
+      if (newRecords.length > 0) {
+        await bulkPut(name, newRecords)
+      }
+
+      if (removedRecords.length > 0) {
+        await bulkDelete(name, removedRecords.map(item => item[adapter.keyPath]))
+      }
+    }),
+  )
+}
+
+export const initStorage = async (adapters: Record<string, IndexedDbAdapter>) => {
+  if (!window.indexedDB) return
+
+  window.addEventListener("beforeunload", () => closeStorage())
+
+  db = await openDB(DB_NAME, DB_VERSION, {
+    upgrade(db: IDBPDatabase) {
+      const names = Object.keys(adapters)
+
+      for (const name of db.objectStoreNames) {
+        if (!names.includes(name)) {
+          db.deleteObjectStore(name)
+        }
+      }
+
+      for (const [name, {keyPath}] of Object.entries(adapters)) {
+        try {
+          db.createObjectStore(name, {keyPath})
+        } catch (e) {
+          console.warn(e)
+        }
+      }
+    },
+  })
+
+  await Promise.all(
+    Object.entries(adapters)
+      .map(([name, config]) => initIndexedDbAdapter(name, config))
+  )
+}
+
+export const closeStorage = async () => {
+  dead.set(true)
+  subs.forEach(unsub => unsub())
+  await db?.close()
+}
+
+export const clearStorage = async () => {
+  await closeStorage()
+  await deleteDB(DB_NAME)
+}

src/app/types.ts

@@ -9,20 +9,11 @@ export type Session = {
   handler?: Nip46Handler
 }
 
-export type Relay = RelayProfile & {
+export type Relay = RelayProfile
-  fetched_at: number,
-}
 
 export type Handle = {
   pubkey: string
   nip05: string
   nip46: string[]
   relays: string[]
-  fetched_at: number,
-}
-
-declare module '@welshman/util' {
-  interface CustomEvent extends TrustedEvent {
-    fetched_at: number
-  }
 }

src/routes/+layout.svelte

@@ -2,6 +2,7 @@
   import "@src/app.css"
   import {onMount} from 'svelte'
   import {page} from "$app/stores"
+  import {createEventStore} from '@welshman/store'
   import {fly} from "@lib/transition"
   import ModalBox from "@lib/components/ModalBox.svelte"
   import Toast from "@app/components/Toast.svelte"
@@ -9,8 +10,11 @@
   import PrimaryNav from "@app/components/PrimaryNav.svelte"
   import SecondaryNav from "@app/components/SecondaryNav.svelte"
   import {modals, clearModal} from "@app/modal"
-  import {session} from "@app/base"
+  import {session, sessions, pk, repository} from "@app/base"
+  import {plaintext, relays, handles} from "@app/state"
+  import {initStorage} from "@app/storage"
 
+  let ready: Promise<void>
   let dialog: HTMLDialogElement
   let prev: any
 
@@ -31,6 +35,21 @@
   }
 
   onMount(() => {
+    ready = initStorage({
+      events: {
+        keyPath: 'id',
+        store: createEventStore(repository),
+      },
+      relays: {
+        keyPath: 'url',
+        store: relays,
+      },
+      handles: {
+        keyPath: 'nip05',
+        store: handles,
+      },
+    })
+
     dialog.addEventListener('close', () => {
       if (modal) {
         clearModal()
@@ -39,26 +58,30 @@
   })
 </script>
 
-<div data-theme="dark">
+{#await ready}
-  <div class="flex h-screen">
+  <div data-theme="dark" />
-    <PrimaryNav />
+{:then}
-    <SecondaryNav />
+  <div data-theme="dark">
-    <div class="flex-grow bg-base-200">
+    <div class="flex h-screen">
-      <slot />
+      <PrimaryNav />
+      <SecondaryNav />
+      <div class="flex-grow bg-base-200">
+        <slot />
+      </div>
     </div>
+    <dialog bind:this={dialog} class="modal modal-bottom sm:modal-middle !z-modal">
+      {#if prev}
+        {#key prev}
+          <ModalBox {...prev} />
+        {/key}
+        <Toast />
+      {/if}
+      {#if $session}
+        <form method="dialog" class="modal-backdrop">
+          <button />
+        </form>
+      {/if}
+    </dialog>
+    <Toast />
   </div>
-  <dialog bind:this={dialog} class="modal modal-bottom sm:modal-middle !z-modal">
+{/await}
-    {#if prev}
-      {#key prev}
-        <ModalBox {...prev} />
-      {/key}
-      <Toast />
-    {/if}
-    {#if $session}
-      <form method="dialog" class="modal-backdrop">
-        <button />
-      </form>
-    {/if}
-  </dialog>
-  <Toast />
-</div>