Add filestorage adapters

This commit is contained in:
Matthew Remmel
2025-09-30 10:31:28 -04:00
committed by hodlbod
parent dd3231e70f
commit 0a8c2faa74
17 changed files with 494 additions and 8 deletions

View File

@@ -11,6 +11,7 @@ apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
dependencies {
implementation project(':capacitor-community-safe-area')
implementation project(':capacitor-app')
implementation project(':capacitor-filesystem')
implementation project(':capacitor-keyboard')
implementation project(':capacitor-preferences')
implementation project(':capacitor-push-notifications')

View File

@@ -8,6 +8,9 @@ project(':capacitor-community-safe-area').projectDir = new File('../node_modules
include ':capacitor-app'
project(':capacitor-app').projectDir = new File('../node_modules/.pnpm/@capacitor+app@7.0.1_@capacitor+core@7.2.0/node_modules/@capacitor/app/android')
include ':capacitor-filesystem'
project(':capacitor-filesystem').projectDir = new File('../node_modules/.pnpm/@capacitor+filesystem@7.1.4_@capacitor+core@7.2.0/node_modules/@capacitor/filesystem/android')
include ':capacitor-keyboard'
project(':capacitor-keyboard').projectDir = new File('../node_modules/.pnpm/@capacitor+keyboard@7.0.1_@capacitor+core@7.2.0/node_modules/@capacitor/keyboard/android')

View File

@@ -8,6 +8,7 @@
/* Begin PBXBuildFile section */
2FAD9763203C412B000D30F8 /* config.xml in Resources */ = {isa = PBXBuildFile; fileRef = 2FAD9762203C412B000D30F8 /* config.xml */; };
3478F0332E846FEB002431E0 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = 3478F0322E846FEB002431E0 /* PrivacyInfo.xcprivacy */; };
50379B232058CBB4000EE86E /* capacitor.config.json in Resources */ = {isa = PBXBuildFile; fileRef = 50379B222058CBB4000EE86E /* capacitor.config.json */; };
504EC3081FED79650016851F /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504EC3071FED79650016851F /* AppDelegate.swift */; };
504EC30D1FED79650016851F /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 504EC30B1FED79650016851F /* Main.storyboard */; };
@@ -21,6 +22,7 @@
051414282E0CC28400BE0BC8 /* Flotilla Chat.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = "Flotilla Chat.entitlements"; sourceTree = "<group>"; };
1F53EE54954731A2328CBC4B /* Pods-Flotilla Chat.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Flotilla Chat.release.xcconfig"; path = "Pods/Target Support Files/Pods-Flotilla Chat/Pods-Flotilla Chat.release.xcconfig"; sourceTree = "<group>"; };
2FAD9762203C412B000D30F8 /* config.xml */ = {isa = PBXFileReference; lastKnownFileType = text.xml; path = config.xml; sourceTree = "<group>"; };
3478F0322E846FEB002431E0 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; lastKnownFileType = text.xml; path = PrivacyInfo.xcprivacy; sourceTree = "<group>"; };
50379B222058CBB4000EE86E /* capacitor.config.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = capacitor.config.json; sourceTree = "<group>"; };
504EC3041FED79650016851F /* Flotilla Chat.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Flotilla Chat.app"; sourceTree = BUILT_PRODUCTS_DIR; };
504EC3071FED79650016851F /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
@@ -58,6 +60,7 @@
504EC2FB1FED79650016851F = {
isa = PBXGroup;
children = (
3478F0322E846FEB002431E0 /* PrivacyInfo.xcprivacy */,
051414282E0CC28400BE0BC8 /* Flotilla Chat.entitlements */,
504EC3061FED79650016851F /* App */,
504EC3051FED79650016851F /* Products */,
@@ -162,6 +165,7 @@
buildActionMask = 2147483647;
files = (
504EC3121FED79650016851F /* LaunchScreen.storyboard in Resources */,
3478F0332E846FEB002431E0 /* PrivacyInfo.xcprivacy in Resources */,
50B271D11FEDC1A000F3C39B /* public in Resources */,
504EC30F1FED79650016851F /* Assets.xcassets in Resources */,
50379B232058CBB4000EE86E /* capacitor.config.json in Resources */,

View File

@@ -13,6 +13,7 @@ def capacitor_pods
pod 'CapacitorCordova', :path => '../../node_modules/.pnpm/@capacitor+ios@7.2.0_@capacitor+core@7.2.0/node_modules/@capacitor/ios'
pod 'CapacitorCommunitySafeArea', :path => '../../node_modules/.pnpm/@capacitor-community+safe-area@7.0.0-alpha.1_@capacitor+core@7.2.0/node_modules/@capacitor-community/safe-area'
pod 'CapacitorApp', :path => '../../node_modules/.pnpm/@capacitor+app@7.0.1_@capacitor+core@7.2.0/node_modules/@capacitor/app'
pod 'CapacitorFilesystem', :path => '../../node_modules/.pnpm/@capacitor+filesystem@7.1.4_@capacitor+core@7.2.0/node_modules/@capacitor/filesystem'
pod 'CapacitorKeyboard', :path => '../../node_modules/.pnpm/@capacitor+keyboard@7.0.1_@capacitor+core@7.2.0/node_modules/@capacitor/keyboard'
pod 'CapacitorPreferences', :path => '../../node_modules/.pnpm/@capacitor+preferences@7.0.2_@capacitor+core@7.2.0/node_modules/@capacitor/preferences'
pod 'CapacitorPushNotifications', :path => '../../node_modules/.pnpm/@capacitor+push-notifications@7.0.1_@capacitor+core@7.2.0/node_modules/@capacitor/push-notifications'

View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSPrivacyAccessedAPITypes</key>
<array>
<dict>
<key>NSPrivacyAccessedAPIType</key>
<string>NSPrivacyAccessedAPICategoryFileTimestamp</string>
<key>NSPrivacyAccessedAPITypeReasons</key>
<array>
<string>C617.1</string>
</array>
</dict>
</array>
</dict>
</plist>

View File

@@ -43,6 +43,7 @@
"@capacitor/app": "^7.0.0",
"@capacitor/cli": "^7.0.0",
"@capacitor/core": "^7.0.1",
"@capacitor/filesystem": "^7.0.0",
"@capacitor/ios": "^7.0.0",
"@capacitor/keyboard": "^7.0.0",
"@capacitor/preferences": "^7.0.2",

BIN
pnpm-lock.yaml generated

Binary file not shown.

View File

@@ -111,7 +111,7 @@ import {
} from "@app/core/state"
import {loadAlertStatuses} from "@app/core/requests"
import {platform, platformName, getPushInfo} from "@app/util/push"
import {preferencesStorageProvider} from "@src/lib/storage"
import {clearFileStorage, preferencesStorageProvider} from "@src/lib/storage"
// Utils
@@ -158,6 +158,8 @@ export const logout = async () => {
localStorage.clear()
await preferencesStorageProvider.clear()
await clearFileStorage()
}
// Synchronization

View File

@@ -1,5 +1,15 @@
import {type StorageProvider} from "@welshman/store"
import {Preferences} from "@capacitor/preferences"
import type {Unsubscriber} from "svelte/store"
import {Encoding, Filesystem, type Directory} from "@capacitor/filesystem"
import {EventsStorageProvider} from "@lib/storage/events"
import {FreshnessStorageProvider} from "@lib/storage/freshness"
import {HandlesStorageProvider} from "@lib/storage/handles"
import {PlaintextStorageProvider} from "@lib/storage/plaintext"
import {RelaysStorageProvider} from "@lib/storage/relays"
import {TrackerStorageProvider} from "@lib/storage/tracker"
import {ZappersStorageProvider} from "@lib/storage/zappers"
import {repository, tracker, unsubscribers} from "@welshman/app"
export class PreferencesStorageProvider implements StorageProvider {
get = async <T>(key: string): Promise<T | undefined> => {
@@ -26,3 +36,60 @@ export class PreferencesStorageProvider implements StorageProvider {
// singleton instance of PreferencesStorageProvider
export const preferencesStorageProvider = new PreferencesStorageProvider()
export interface FilesystemStorageProvider {
initializeState(): Promise<void>
sync(): Unsubscriber
}
export const getAllFromFile = async <T>(
filepath: string,
directory: Directory,
encoding: Encoding,
): Promise<T[]> => {
try {
const contents = (
await Filesystem.readFile({
path: filepath,
directory,
encoding,
})
).data.toString()
if (!contents || contents == "") {
return []
}
return JSON.parse(contents)
} catch (err) {
// file doesn't exist
return []
}
}
export const defaultStorageProviders = {
relays: new RelaysStorageProvider(),
handles: new RelaysStorageProvider(),
zappers: new ZappersStorageProvider(),
freshness: new FreshnessStorageProvider(),
plaintext: new PlaintextStorageProvider(),
tracker: new TrackerStorageProvider({tracker}),
events: new EventsStorageProvider({limit: 10_000, repository, rankEvent: () => 1}),
}
export const initFileStorage = async (storageProviders: Record<string, FilesystemStorageProvider>) => {
await Promise.all(Object.values(storageProviders).map(async provider => {
await provider.initializeState()
unsubscribers.push(provider.sync())
}))
}
export const clearFileStorage = async (): Promise<void> => {
await EventsStorageProvider.clearStorage()
await FreshnessStorageProvider.clearStorage()
await HandlesStorageProvider.clearStorage()
await PlaintextStorageProvider.clearStorage()
await RelaysStorageProvider.clearStorage()
await TrackerStorageProvider.clearStorage()
await ZappersStorageProvider.clearStorage()
}

102
src/lib/storage/events.ts Normal file
View File

@@ -0,0 +1,102 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import type {TrustedEvent} from "@welshman/util"
import type {Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import type {Repository, RepositoryUpdate} from "@welshman/relay"
import {on, sortBy} from "@welshman/lib"
export class EventsStorageProvider implements FilesystemStorageProvider {
static filepath = "events.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
limit: number
repository: Repository
rankEvent: (event: TrustedEvent) => number
eventCount: number = 0
isDeleting = false
constructor({
limit,
repository,
rankEvent,
}: {
limit: number
repository: Repository
rankEvent: (event: TrustedEvent) => number
}) {
this.limit = limit
this.repository = repository
this.rankEvent = rankEvent
}
async initializeState(): Promise<void> {
const events = await this.getAll()
this.eventCount = events.length
this.repository.load(events)
}
sync(): Unsubscriber {
const onUpdate = async ({added, removed}: RepositoryUpdate) => {
// Only add events we want to keep
const keep = added.filter(e => this.rankEvent(e) > 0)
// Add new events
if (keep.length > 0) {
await this.updateEvents(keep)
}
// If we're well above our retention limit, drop lowest-ranked events
if (!this.isDeleting && this.eventCount > this.limit * 1.5) {
try {
this.isDeleting = true
for (const event of sortBy(e => -this.rankEvent(e), await this.getAll()).slice(
this.limit,
)) {
removed.add(event.id)
}
if (removed.size > 0) {
await this.deleteEvents(Array.from(removed))
}
} finally {
this.isDeleting = false
}
}
// Keep track of our total number of events. This isn't strictly accurate, but it's close enough
this.eventCount = this.eventCount + keep.length - removed.size
}
return on(this.repository, "update", onUpdate)
}
async getAll(): Promise<TrustedEvent[]> {
return await getAllFromFile(EventsStorageProvider.filepath, EventsStorageProvider.directory, EventsStorageProvider.encoding)
}
async writeAll(events: TrustedEvent[]) {
await Filesystem.writeFile({
path: EventsStorageProvider.filepath,
directory: EventsStorageProvider.directory,
encoding: EventsStorageProvider.encoding,
data: JSON.stringify(events),
})
}
async updateEvents(events: TrustedEvent[]) {
const existing = await this.getAll()
const updated = existing.concat(events)
await this.writeAll(updated)
}
async deleteEvents(ids: string[]) {
const existing = await this.getAll()
const updated = existing.filter(e => !ids.includes(e.id))
await this.writeAll(updated)
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: EventsStorageProvider.filepath, directory: EventsStorageProvider.directory})
}
}

View File

@@ -0,0 +1,45 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import type {Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import {fromPairs} from "@welshman/lib"
import {freshness} from "@welshman/store"
type KV = {key: string; value: any}
export class FreshnessStorageProvider implements FilesystemStorageProvider {
static filepath = "freshness.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
async initializeState(): Promise<void> {
const items = await this.getAll()
freshness.set(fromPairs(items.map(item => [item.key, item.value])))
}
sync(): Unsubscriber {
const interval = setInterval(() => {
this.writeAll(freshness.get())
}, 10_000)
return () => clearInterval(interval)
}
async getAll(): Promise<KV[]> {
return await getAllFromFile(FreshnessStorageProvider.filepath, FreshnessStorageProvider.directory, FreshnessStorageProvider.encoding)
}
async writeAll(items: Record<string, any>) {
const kvs = Object.entries(items).map(([key, value]) => ({key, value}))
await Filesystem.writeFile({
path: FreshnessStorageProvider.filepath,
directory: FreshnessStorageProvider.directory,
encoding: FreshnessStorageProvider.encoding,
data: JSON.stringify(kvs),
})
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: FreshnessStorageProvider.filepath, directory: FreshnessStorageProvider.directory})
}
}

View File

@@ -0,0 +1,40 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import {get, type Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import {batch} from "@welshman/lib"
import {handles, onHandle, type Handle} from "@welshman/app"
export class HandlesStorageProvider implements FilesystemStorageProvider {
static filepath = "handles.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
async initializeState(): Promise<void> {
handles.set(await this.getAll())
}
sync(): Unsubscriber {
return onHandle(batch(300, () => this.saveState()))
}
async getAll(): Promise<Handle[]> {
return await getAllFromFile(HandlesStorageProvider.filepath, HandlesStorageProvider.directory, HandlesStorageProvider.encoding)
}
async writeAll(handles: Handle[]) {
await Filesystem.writeFile({
path: HandlesStorageProvider.filepath,
directory: HandlesStorageProvider.directory,
encoding: HandlesStorageProvider.encoding,
data: JSON.stringify(handles),
})
}
async saveState() {
await this.writeAll(get(handles))
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: HandlesStorageProvider.filepath, directory: HandlesStorageProvider.directory})
}
}

View File

@@ -0,0 +1,45 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import type {Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import {fromPairs} from "@welshman/lib"
import {plaintext} from "@welshman/app"
type KV = {key: string; value: any}
export class PlaintextStorageProvider implements FilesystemStorageProvider {
static filepath = "plaintext.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
async initializeState(): Promise<void> {
const items = await this.getAll()
plaintext.set(fromPairs(items.map(item => [item.key, item.value])))
}
sync(): Unsubscriber {
const interval = setInterval(() => {
this.writeAll(plaintext.get())
}, 10_000)
return () => clearInterval(interval)
}
async getAll(): Promise<KV[]> {
return await getAllFromFile(PlaintextStorageProvider.filepath, PlaintextStorageProvider.directory, PlaintextStorageProvider.encoding)
}
async writeAll(items: Record<string, any>) {
const kvs = Object.entries(items).map(([key, value]) => ({key, value}))
await Filesystem.writeFile({
path: PlaintextStorageProvider.filepath,
directory: PlaintextStorageProvider.directory,
encoding: PlaintextStorageProvider.encoding,
data: JSON.stringify(kvs),
})
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: PlaintextStorageProvider.filepath, directory: PlaintextStorageProvider.directory})
}
}

40
src/lib/storage/relays.ts Normal file
View File

@@ -0,0 +1,40 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import type {Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import {relays, type Relay} from "@welshman/app"
import {throttled} from "@welshman/store"
export class RelaysStorageProvider implements FilesystemStorageProvider {
static filepath = "relays.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
async initializeState(): Promise<void> {
relays.set(await this.getAll())
}
sync(): Unsubscriber {
return throttled(3000, relays).subscribe(() => this.saveState())
}
async getAll(): Promise<Relay[]> {
return await getAllFromFile(RelaysStorageProvider.filepath, RelaysStorageProvider.directory, RelaysStorageProvider.encoding)
}
async writeAll(relays: Relay[]) {
await Filesystem.writeFile({
path: RelaysStorageProvider.filepath,
directory: RelaysStorageProvider.directory,
encoding: RelaysStorageProvider.encoding,
data: JSON.stringify(relays),
})
}
async saveState() {
await this.writeAll(relays.get())
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: RelaysStorageProvider.filepath, directory: RelaysStorageProvider.directory})
}
}

View File

@@ -0,0 +1,80 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import type {Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import type {Tracker} from "@welshman/net"
import {call, on} from "@welshman/lib"
type Entry = {id: string; relays: string[]}
export class TrackerStorageProvider implements FilesystemStorageProvider {
static filepath = "tracker.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
tracker: Tracker
constructor({tracker}: {tracker: Tracker}) {
this.tracker = tracker
}
async initializeState(): Promise<void> {
const relaysByid = new Map<string, Set<string>>()
for (const {id, relays} of await this.getAll()) {
relaysByid.set(id, new Set(relays))
}
this.tracker.load(relaysByid)
}
sync(): Unsubscriber {
const updateOne = async (id: string, relay: string) => {
const relays = new Set(await this.getAll())
relays.add({id, relays: Array.from(this.tracker.getRelays(id))})
await this.writeAll([...relays])
}
const updateAll = async () => {
await this.writeAll(Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({
id,
relays: Array.from(relays),
})))
}
const unsubscribers = [
on(this.tracker, "add", updateOne),
on(this.tracker, "remove", updateOne),
on(this.tracker, "load", updateAll),
on(this.tracker, "clear", updateAll),
]
return () => {
unsubscribers.forEach(call)
}
}
async getAll(): Promise<Entry[]> {
return await getAllFromFile(TrackerStorageProvider.filepath, TrackerStorageProvider.directory, TrackerStorageProvider.encoding)
}
async writeAll(relays: Entry[]) {
await Filesystem.writeFile({
path: TrackerStorageProvider.filepath,
directory: TrackerStorageProvider.directory,
encoding: TrackerStorageProvider.encoding,
data: JSON.stringify(relays),
})
}
async saveState() {
await this.writeAll(
Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({
id,
relays: Array.from(relays),
})),
)
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: TrackerStorageProvider.filepath, directory: TrackerStorageProvider.directory})
}
}

View File

@@ -0,0 +1,41 @@
import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage"
import {get, type Unsubscriber} from "svelte/store"
import {Filesystem, Directory, Encoding} from "@capacitor/filesystem"
import {onZapper, zappers} from "@welshman/app"
import {batch} from "@welshman/lib"
import type {Zapper} from "@welshman/util"
export class ZappersStorageProvider implements FilesystemStorageProvider {
static filepath = "zappers.json"
static directory = Directory.Data
static encoding = Encoding.UTF8
async initializeState(): Promise<void> {
zappers.set(await this.getAll())
}
sync(): Unsubscriber {
return onZapper(batch(300, () => this.saveState()))
}
async getAll(): Promise<Zapper[]> {
return await getAllFromFile(ZappersStorageProvider.filepath, ZappersStorageProvider.directory, ZappersStorageProvider.encoding)
}
async writeAll(zappers: Zapper[]) {
await Filesystem.writeFile({
path: ZappersStorageProvider.filepath,
directory: ZappersStorageProvider.directory,
encoding: ZappersStorageProvider.encoding,
data: JSON.stringify(zappers),
})
}
async saveState() {
await this.writeAll(get(zappers))
}
static async clearStorage(): Promise<void> {
await Filesystem.deleteFile({path: ZappersStorageProvider.filepath, directory: ZappersStorageProvider.directory})
}
}

View File

@@ -62,7 +62,6 @@
import {
loadRelay,
db,
initStorage,
repository,
pubkey,
session,
@@ -70,7 +69,6 @@
signer,
signerLog,
dropSession,
defaultStorageAdapters,
loginWithNip01,
loginWithNip46,
EventsStorageAdapter,
@@ -85,7 +83,7 @@
import * as net from "@welshman/net"
import * as app from "@welshman/app"
import {nsecDecode} from "@lib/util"
import {preferencesStorageProvider} from "@lib/storage"
import {defaultStorageProviders, initFileStorage, preferencesStorageProvider} from "@lib/storage"
import AppContainer from "@app/components/AppContainer.svelte"
import ModalContainer from "@app/components/ModalContainer.svelte"
import {setupTracking} from "@app/util/tracking"
@@ -111,6 +109,7 @@
import * as appState from "@app/core/state"
import {badgeCount, handleBadgeCountChanges} from "@app/util/notifications"
import NewNotificationSound from "@src/app/components/NewNotificationSound.svelte"
import {EventsStorageProvider} from "@lib/storage/events"
// Migration: old nostrtalk instance used different sessions
if ($session && !$signer) {
@@ -280,10 +279,8 @@
storage: preferencesStorageProvider,
})
await initStorage("flotilla", 8, {
...defaultStorageAdapters,
events: new EventsStorageAdapter({
name: "events",
await initFileStorage({...defaultStorageProviders,
events: new EventsStorageProvider({
limit: 10_000,
repository,
rankEvent: (e: TrustedEvent) => {