From 697e893ae6df1bfce451d6c17794b645c10da17e Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Wed, 14 Aug 2024 12:46:25 -0700 Subject: [PATCH] Add indexeddb storage --- package-lock.json | 6 + package.json | 1 + src/app/base.ts | 2 +- src/app/components/LogIn.svelte | 1 + src/app/state.ts | 209 ++++++++++++++++++-------------- src/app/storage.ts | 123 +++++++++++++++++++ src/app/types.ts | 11 +- src/routes/+layout.svelte | 67 ++++++---- 8 files changed, 296 insertions(+), 124 deletions(-) create mode 100644 src/app/storage.ts diff --git a/package-lock.json b/package-lock.json index 5d62248..cd6bc28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@welshman/store": "^0.0.2", "@welshman/util": "^0.0.25", "daisyui": "^4.12.10", + "idb": "^8.0.0", "nostr-tools": "^2.7.2", "prettier-plugin-tailwindcss": "^0.6.5", "throttle-debounce": "^5.0.2" @@ -2724,6 +2725,11 @@ "node": ">= 0.4" } }, + "node_modules/idb": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.0.tgz", + "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==" + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", diff --git a/package.json b/package.json index 37f23b6..bdec970 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@welshman/store": "^0.0.2", "@welshman/util": "^0.0.25", "daisyui": "^4.12.10", + "idb": "^8.0.0", "nostr-tools": "^2.7.2", "prettier-plugin-tailwindcss": "^0.6.5", "throttle-debounce": "^5.0.2" diff --git a/src/app/base.ts b/src/app/base.ts index d3ddeb6..2275eb1 100644 --- a/src/app/base.ts +++ b/src/app/base.ts @@ -1,4 +1,4 @@ -import {derived} from "svelte/store" +import {derived, writable} from "svelte/store" import {memoize, assoc} from '@welshman/lib' import type {CustomEvent} from '@welshman/util' import {Repository, createEvent, Relay} from "@welshman/util" diff --git a/src/app/components/LogIn.svelte b/src/app/components/LogIn.svelte index 06d510e..7bc1be3 100644 --- a/src/app/components/LogIn.svelte +++ b/src/app/components/LogIn.svelte @@ -17,6 +17,7 @@ const tryLogin = async () => { const secret = makeSecret() const handle = await loadHandle(`${username}@${handler.domain}`) + console.log(handle) if (!handle?.pubkey) { return pushToast({ diff --git a/src/app/state.ts b/src/app/state.ts index b10f5aa..8409bb0 100644 --- a/src/app/state.ts +++ b/src/app/state.ts @@ -8,36 +8,46 @@ import type {SubscribeRequest} from '@welshman/net' import {publish, subscribe} from '@welshman/net' import {decrypt} from '@welshman/signer' import {deriveEvents, deriveEventsMapped, getter, withGetter} from "@welshman/store" -import {synced, parseJson} from '@lib/util' +import {parseJson} from '@lib/util' import type {Session, Handle, Relay} from '@app/types' import {INDEXER_RELAYS, DUFFLEPUD_URL, repository, pk, getSession, getSigner, signer} from "@app/base" // Utils export const createCollection = ({ + name, store, getKey, - isStale, load, }: { + name: string, store: Readable, getKey: (item: T) => string, - isStale: (item: T) => boolean, load: (key: string, ...args: any) => Promise }) => { const indexStore = derived(store, $items => indexBy(getKey, $items)) const getIndex = getter(indexStore) - const getItem = (key: string) => getIndex().get(key) + const pending = new Map>> const loadItem = async (key: string, ...args: any[]) => { - const item = getIndex().get(key) - - if (item && !isStale(item)) { - return item + if (getFreshness(name, key) > now() - 3600) { + return getIndex().get(key) } - await load(key, ...args) + if (pending.has(key)) { + await pending.get(key) + } else { + setFreshness(name, key, now()) + + const promise = load(key, ...args) + + pending.set(key, promise) + + await promise + + pending.delete(key) + } return getIndex().get(key) } @@ -61,9 +71,7 @@ export const load = (request: SubscribeRequest) => new Promise>(resolve => { const sub = subscribe({closeOnEose: true, timeout: 3000, delay: 50, ...request}) - sub.emitter.on('event', (url: string, event: SignedEvent) => { - const e: CustomEvent = {...event, fetched_at: now()} - + sub.emitter.on('event', (url: string, e: SignedEvent) => { repository.publish(e) sub.close() resolve(e) @@ -104,6 +112,25 @@ export const updateList = async (kind: number, modifyTags: ModifyTags) => { await publish({event, relays}) } +// Freshness + +export const freshness = withGetter(writable>({})) + +export const getFreshnessKey = (ns: string, key: string) => `${ns}:${key}` + +export const getFreshness = (ns: string, key: string) => freshness.get()[getFreshnessKey(ns, key)] || 0 + +export const setFreshness = (ns: string, key: string, ts: number) => freshness.update(assoc(getFreshnessKey(ns, key), ts)) + +export const setFreshnessBulk = (ns: string, updates: Record) => + freshness.update($freshness => { + for (const [key, ts] of Object.entries(updates)) { + $freshness[key] = ts + } + + return $freshness + }) + // Plaintext export const plaintext = withGetter(writable>({})) @@ -138,23 +165,23 @@ export const { loadItem: loadRelay, // getItem: getRelay, } = createCollection({ - store: relays, - getKey: (relay: Relay) => relay.url, - isStale: (relay: Relay) => relay.fetched_at < now() - 3600, - load: batcher(800, async (urls: string[]) => { - const urlSet = new Set(urls) - const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)}) - const index = indexBy((item: any) => item.url, res?.data || []) - const items: Relay[] = urls.map(url => { - const {info = {}} = index.get(url) || {} + name: 'relays', + store: relays, + getKey: (relay: Relay) => relay.url, + load: batcher(800, async (urls: string[]) => { + const urlSet = new Set(urls) + const res = await postJson(`${DUFFLEPUD_URL}/relay/info`, {urls: Array.from(urlSet)}) + const index = indexBy((item: any) => item.url, res?.data || []) + const items: Relay[] = urls.map(url => { + const {info = {}} = index.get(url) || {} - return {...info, url, fetched_at: now()} - }) + return {...info, url} + }) - relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items])) + relays.update($relays => uniqBy($relay => $relay.url, [...$relays, ...items])) - return items - }), + return items + }), }) // Handles @@ -162,29 +189,29 @@ export const { export const handles = writable([]) export const { - indexStore: handlesByPubkey, - getIndex: getHandlesByPubkey, + indexStore: handlesByNip05, + getIndex: getHandlesByNip05, deriveItem: deriveHandle, loadItem: loadHandle, // getItem: getHandle, } = createCollection({ - store: handles, - getKey: (handle: Handle) => handle.pubkey, - isStale: (handle: Handle) => handle.fetched_at < now() - 3600, - load: batcher(800, async (nip05s: string[]) => { - const nip05Set = new Set(nip05s) - const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)}) - const index = indexBy((item: any) => item.handle, res?.data || []) - const items: Handle[] = nip05s.map(nip05 => { - const {info = {}} = index.get(nip05) || {} + name: 'handles', + store: handles, + getKey: (handle: Handle) => handle.nip05, + load: batcher(800, async (nip05s: string[]) => { + const nip05Set = new Set(nip05s) + const res = await postJson(`${DUFFLEPUD_URL}/handle/info`, {handles: Array.from(nip05Set)}) + const index = indexBy((item: any) => item.handle, res?.data || []) + const items: Handle[] = nip05s.map(nip05 => { + const {info = {}} = index.get(nip05) || {} - return {...info, nip05, fetched_at: now()} - }) + return {...info, nip05} + }) - handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items])) + handles.update($handles => uniqBy($handle => $handle.nip05, [...$handles, ...items])) - return items - }), + return items + }), }) // Profiles @@ -203,15 +230,15 @@ export const { loadItem: loadProfile, // getItem: getProfile, } = createCollection({ - store: profiles, - getKey: profile => profile.event.pubkey, - isStale: (profile: PublishedProfile) => profile.event.fetched_at < now() - 3600, - load: (pubkey: string, relays = [], request: Partial = {}) => - load({ - ...request, - relays: [...relays, ...INDEXER_RELAYS], - filters: [{kinds: [PROFILE], authors: [pubkey]}], - }), + name: 'profiles', + store: profiles, + getKey: profile => profile.event.pubkey, + load: (pubkey: string, relays = [], request: Partial = {}) => + load({ + ...request, + relays: [...relays, ...INDEXER_RELAYS], + filters: [{kinds: [PROFILE], authors: [pubkey]}], + }), }) // Relay selections @@ -231,15 +258,15 @@ export const { loadItem: loadRelaySelections, // getItem: getRelaySelections, } = createCollection({ - store: relaySelections, - getKey: relaySelections => relaySelections.pubkey, - isStale: (relaySelections: CustomEvent) => relaySelections.fetched_at < now() - 3600, - load: (pubkey: string, relays = [], request: Partial = {}) => - load({ - ...request, - relays: [...relays, ...INDEXER_RELAYS], - filters: [{kinds: [RELAYS], authors: [pubkey]}], - }) + name: 'relaySelections', + store: relaySelections, + getKey: relaySelections => relaySelections.pubkey, + load: (pubkey: string, relays = [], request: Partial = {}) => + load({ + ...request, + relays: [...relays, ...INDEXER_RELAYS], + filters: [{kinds: [RELAYS], authors: [pubkey]}], + }) }) // Follows @@ -263,15 +290,15 @@ export const { loadItem: loadFollows, // getItem: getFollows, } = createCollection({ - store: follows, - getKey: follows => follows.event.pubkey, - isStale: (follows: PublishedList) => follows.event.fetched_at < now() - 3600, - load: (pubkey: string, relays = [], request: Partial = {}) => - load({ - ...request, - relays: [...relays, ...INDEXER_RELAYS], - filters: [{kinds: [FOLLOWS], authors: [pubkey]}], - }) + name: 'follows', + store: follows, + getKey: follows => follows.event.pubkey, + load: (pubkey: string, relays = [], request: Partial = {}) => + load({ + ...request, + relays: [...relays, ...INDEXER_RELAYS], + filters: [{kinds: [FOLLOWS], authors: [pubkey]}], + }) }) // Mutes @@ -295,15 +322,15 @@ export const { loadItem: loadMutes, // getItem: getMutes, } = createCollection({ - store: mutes, - getKey: mute => mute.event.pubkey, - isStale: (mutes: PublishedList) => mutes.event.fetched_at < now() - 3600, - load: (pubkey: string, relays = [], request: Partial = {}) => - load({ - ...request, - relays: [...relays, ...INDEXER_RELAYS], - filters: [{kinds: [MUTES], authors: [pubkey]}], - }) + name: 'mutes', + store: mutes, + getKey: mute => mute.event.pubkey, + load: (pubkey: string, relays = [], request: Partial = {}) => + load({ + ...request, + relays: [...relays, ...INDEXER_RELAYS], + filters: [{kinds: [MUTES], authors: [pubkey]}], + }) }) // Groups @@ -360,18 +387,18 @@ export const { loadItem: loadGroup, // getItem: getGroup, } = createCollection({ - store: groups, - getKey: (group: PublishedGroup) => group.nom, - isStale: (group: PublishedGroup) => group.event.fetched_at < now() - 3600, - load: (nom: string, relays: string[] = [], request: Partial = {}) => - Promise.all([ - ...relays.map(loadRelay), - load({ - ...request, - relays, - filters: [{kinds: [GROUP_META], '#d': [nom]}], - }), - ]) + name: 'groups', + store: groups, + getKey: (group: PublishedGroup) => group.nom, + load: (nom: string, relays: string[] = [], request: Partial = {}) => + Promise.all([ + ...relays.map(loadRelay), + load({ + ...request, + relays, + filters: [{kinds: [GROUP_META], '#d': [nom]}], + }), + ]) }) // Qualified groups @@ -433,9 +460,9 @@ export const { loadItem: loadGroupMembership, // getItem: getGroupMembership, } = createCollection({ + name: 'groupMemberships', store: groupMemberships, getKey: groupMembership => groupMembership.event.pubkey, - isStale: (groupMembership: PublishedGroupMembership) => groupMembership.event.fetched_at < now() - 3600, load: (pubkey: string, relays = [], request: Partial = {}) => load({ ...request, diff --git a/src/app/storage.ts b/src/app/storage.ts new file mode 100644 index 0000000..34746cb --- /dev/null +++ b/src/app/storage.ts @@ -0,0 +1,123 @@ +import {openDB, deleteDB} from "idb" +import type {IDBPDatabase} from "idb" +import {throttle} from 'throttle-debounce' +import {writable} from 'svelte/store' +import type {Unsubscriber, Writable} from 'svelte/store' +import {isNil, randomInt} from '@welshman/lib' +import {withGetter} from '@welshman/store' +import {getJson, setJson} from '@lib/util' +import {pk, sessions, repository} from '@app/base' + +export type Item = Record + +export type IndexedDbAdapter = { + keyPath: string + store: Writable +} + +export let db: IDBPDatabase + +export const dead = withGetter(writable(false)) + +export const subs: Unsubscriber[] = [] + +export const DB_NAME = "flotilla" + +export const DB_VERSION = 1 + +export const getAll = async (name: string) => { + const tx = db.transaction(name, "readwrite") + const store = tx.objectStore(name) + const result = await store.getAll() + + await tx.done + + return result +} + +export const bulkPut = async (name: string, data: any[]) => { + const tx = db.transaction(name, "readwrite") + const store = tx.objectStore(name) + + await Promise.all(data.map(item => store.put(item))) + await tx.done +} + +export const bulkDelete = async (name: string, ids: string[]) => { + const tx = db.transaction(name, "readwrite") + const store = tx.objectStore(name) + + await Promise.all(ids.map(id => store.delete(id))) + await tx.done +} + +export const initIndexedDbAdapter = async (name: string, adapter: IndexedDbAdapter) => { + let copy = await getAll(name) + + adapter.store.set(copy) + + adapter.store.subscribe( + throttle(randomInt(3000, 5000), async (data: Item[]) => { + if (dead.get()) { + return + } + + const prevIds = new Set(copy.map(item => item[adapter.keyPath])) + const currentIds = new Set(data.map(item => item[adapter.keyPath])) + const newRecords = data.filter(r => !prevIds.has(r[adapter.keyPath])) + const removedRecords = copy.filter(r => !currentIds.has(r[adapter.keyPath])) + + copy = data + + if (newRecords.length > 0) { + await bulkPut(name, newRecords) + } + + if (removedRecords.length > 0) { + await bulkDelete(name, removedRecords.map(item => item[adapter.keyPath])) + } + }), + ) +} + +export const initStorage = async (adapters: Record) => { + if (!window.indexedDB) return + + window.addEventListener("beforeunload", () => closeStorage()) + + db = await openDB(DB_NAME, DB_VERSION, { + upgrade(db: IDBPDatabase) { + const names = Object.keys(adapters) + + for (const name of db.objectStoreNames) { + if (!names.includes(name)) { + db.deleteObjectStore(name) + } + } + + for (const [name, {keyPath}] of Object.entries(adapters)) { + try { + db.createObjectStore(name, {keyPath}) + } catch (e) { + console.warn(e) + } + } + }, + }) + + await Promise.all( + Object.entries(adapters) + .map(([name, config]) => initIndexedDbAdapter(name, config)) + ) +} + +export const closeStorage = async () => { + dead.set(true) + subs.forEach(unsub => unsub()) + await db?.close() +} + +export const clearStorage = async () => { + await closeStorage() + await deleteDB(DB_NAME) +} diff --git a/src/app/types.ts b/src/app/types.ts index e596949..9d13c97 100644 --- a/src/app/types.ts +++ b/src/app/types.ts @@ -9,20 +9,11 @@ export type Session = { handler?: Nip46Handler } -export type Relay = RelayProfile & { - fetched_at: number, -} +export type Relay = RelayProfile export type Handle = { pubkey: string nip05: string nip46: string[] relays: string[] - fetched_at: number, -} - -declare module '@welshman/util' { - interface CustomEvent extends TrustedEvent { - fetched_at: number - } } diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 862c1e1..73e2192 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -2,6 +2,7 @@ import "@src/app.css" import {onMount} from 'svelte' import {page} from "$app/stores" + import {createEventStore} from '@welshman/store' import {fly} from "@lib/transition" import ModalBox from "@lib/components/ModalBox.svelte" import Toast from "@app/components/Toast.svelte" @@ -9,8 +10,11 @@ import PrimaryNav from "@app/components/PrimaryNav.svelte" import SecondaryNav from "@app/components/SecondaryNav.svelte" import {modals, clearModal} from "@app/modal" - import {session} from "@app/base" + import {session, sessions, pk, repository} from "@app/base" + import {plaintext, relays, handles} from "@app/state" + import {initStorage} from "@app/storage" + let ready: Promise let dialog: HTMLDialogElement let prev: any @@ -31,6 +35,21 @@ } onMount(() => { + ready = initStorage({ + events: { + keyPath: 'id', + store: createEventStore(repository), + }, + relays: { + keyPath: 'url', + store: relays, + }, + handles: { + keyPath: 'nip05', + store: handles, + }, + }) + dialog.addEventListener('close', () => { if (modal) { clearModal() @@ -39,26 +58,30 @@ }) -
-
- - -
- +{#await ready} +
+{:then} +
+
+ + +
+ +
+ + {#if prev} + {#key prev} + + {/key} + + {/if} + {#if $session} +
+
+
- - {#if prev} - {#key prev} - - {/key} - - {/if} - {#if $session} -
-
- -
+{/await}