From e9173a68949da475a2f5366fa135ef11706ccb01 Mon Sep 17 00:00:00 2001 From: mleku Date: Fri, 5 Dec 2025 14:42:22 +0000 Subject: [PATCH] Update event import process and improve user feedback Simplified event import to run synchronously, ensuring proper resource handling and accurate feedback. Enhanced frontend with real-time import status messages and error handling. Adjusted migrations to handle events individually, improving reliability and granular progress tracking. --- .claude/settings.local.json | 3 +- app/web/src/App.svelte | 21 ++++++-- app/web/src/ImportView.svelte | 44 +++++++++++++--- pkg/database/import.go | 11 ++-- pkg/database/migrations.go | 99 ++++++++++++++++------------------- pkg/version/version | 2 +- 6 files changed, 108 insertions(+), 72 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 4205c61..c3e6282 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -12,7 +12,8 @@ "Write:*", "Bash(go build:*)", "Bash(go test:*)", - "Bash(./scripts/test.sh:*)" + "Bash(./scripts/test.sh:*)", + "Bash(./scripts/update-embedded-web.sh:*)" ], "deny": [], "ask": [] diff --git a/app/web/src/App.svelte b/app/web/src/App.svelte index ce6534d..7504e1b 100644 --- a/app/web/src/App.svelte +++ b/app/web/src/App.svelte @@ -53,6 +53,7 @@ let searchTabs = []; let allEvents = []; let selectedFile = null; + let importMessage = ""; // Message shown after import completes let expandedEvents = new Set(); let isLoadingEvents = false; let hasMoreEvents = true; @@ -2356,22 +2357,28 @@ // Import functionality function handleFileSelect(event) { - selectedFile = event.target.files[0]; + // event.detail contains the original DOM event from the child component + selectedFile = event.detail.target.files[0]; } async function importEvents() { // Skip login/permission check when ACL is "none" (open relay mode) if (aclMode !== "none" && (!isLoggedIn || (userRole !== "admin" && userRole !== "owner"))) { - alert("Admin or owner permission required"); + importMessage = "Admin or owner permission required"; + setTimeout(() => { importMessage = ""; }, 5000); return; } if (!selectedFile) { - alert("Please select a file"); + importMessage = "Please select a file"; + setTimeout(() => { importMessage = ""; }, 5000); return; } try { + // Show uploading message + importMessage = "Uploading..."; + // Build headers - only include auth when ACL is not "none" const headers = {}; if (aclMode !== "none" && isLoggedIn) { @@ -2396,12 +2403,15 @@ } const result = await response.json(); - alert("Import started successfully"); + importMessage = "Upload complete"; selectedFile = null; document.getElementById("import-file").value = ""; + // Clear message after 5 seconds + setTimeout(() => { importMessage = ""; }, 5000); } catch (error) { console.error("Import failed:", error); - alert("Import failed: " + error.message); + importMessage = "Import failed: " + error.message; + setTimeout(() => { importMessage = ""; }, 5000); } } @@ -2952,6 +2962,7 @@ {currentEffectiveRole} {selectedFile} {aclMode} + {importMessage} on:fileSelect={handleFileSelect} on:importEvents={importEvents} on:openLoginModal={openLoginModal} diff --git a/app/web/src/ImportView.svelte b/app/web/src/ImportView.svelte index 48a191d..b55047b 100644 --- a/app/web/src/ImportView.svelte +++ b/app/web/src/ImportView.svelte @@ -3,6 +3,7 @@ export let currentEffectiveRole = ""; export let selectedFile = null; export let aclMode = ""; + export let importMessage = ""; import { createEventDispatcher } from "svelte"; const dispatch = createEventDispatcher(); @@ -34,13 +35,18 @@ accept=".jsonl,.txt" on:change={handleFileSelect} /> - +
+ + {#if importMessage} + {importMessage} + {/if} +
{:else if isLoggedIn}
@@ -122,6 +128,30 @@ cursor: not-allowed; } + .import-row { + display: flex; + align-items: center; + gap: 1em; + } + + .import-message { + font-size: 0.9em; + padding: 0.25em 0.5em; + border-radius: 0.25em; + } + + .import-message.uploading { + color: var(--primary); + } + + .import-message.success { + color: #4caf50; + } + + .import-message.error { + color: #f44336; + } + .permission-denied, .login-prompt { text-align: center; diff --git a/pkg/database/import.go b/pkg/database/import.go index af2201e..78fe9af 100644 --- a/pkg/database/import.go +++ b/pkg/database/import.go @@ -10,10 +10,11 @@ import ( ) // Import a collection of events in line structured minified JSON format (JSONL). +// This runs synchronously to ensure the reader remains valid during processing. +// The actual event processing happens after buffering to a temp file, so the +// caller can close the reader after Import returns. func (d *D) Import(rr io.Reader) { - go func() { - if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) { - log.E.F("import failed: %v", err) - } - }() + if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) { + log.E.F("import failed: %v", err) + } } diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index f3ffa96..a6f390e 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -850,66 +850,59 @@ func (d *D) ConvertToCompactEventFormat() { return } - // Second pass: convert in batches - const batchSize = 500 - for i := 0; i < len(migrations); i += batchSize { - end := i + batchSize - if end > len(migrations) { - end = len(migrations) - } - batch := migrations[i:end] - + // Process each event individually to avoid transaction size limits + // Some events (like kind 3 follow lists) can be very large + for i, m := range migrations { if err = d.Update(func(txn *badger.Txn) error { - for _, m := range batch { - // Decode the legacy event - ev := new(event.E) - if err = ev.UnmarshalBinary(bytes.NewBuffer(m.OldData)); chk.E(err) { - log.W.F("migration: failed to decode event serial %d: %v", m.Serial, err) - continue - } - - // Store SerialEventId mapping - if err = d.StoreEventIdSerial(txn, m.Serial, m.EventId); chk.E(err) { - log.W.F("migration: failed to store event ID mapping for serial %d: %v", m.Serial, err) - continue - } - - // Encode in compact format - compactData, encErr := MarshalCompactEvent(ev, resolver) - if encErr != nil { - log.W.F("migration: failed to encode compact event for serial %d: %v", m.Serial, encErr) - continue - } - - // Store compact event - ser := new(types.Uint40) - if err = ser.Set(m.Serial); chk.E(err) { - continue - } - cmpKey := new(bytes.Buffer) - if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); chk.E(err) { - continue - } - if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) { - log.W.F("migration: failed to store compact event for serial %d: %v", m.Serial, err) - continue - } - - // Track savings - savedBytes += int64(len(m.OldData) - len(compactData)) - processedCount++ - - // Cache the mappings - d.serialCache.CacheEventId(m.Serial, m.EventId) + // Decode the legacy event + ev := new(event.E) + if err = ev.UnmarshalBinary(bytes.NewBuffer(m.OldData)); chk.E(err) { + log.W.F("migration: failed to decode event serial %d: %v", m.Serial, err) + return nil // Continue with next event } + + // Store SerialEventId mapping + if err = d.StoreEventIdSerial(txn, m.Serial, m.EventId); chk.E(err) { + log.W.F("migration: failed to store event ID mapping for serial %d: %v", m.Serial, err) + return nil // Continue with next event + } + + // Encode in compact format + compactData, encErr := MarshalCompactEvent(ev, resolver) + if encErr != nil { + log.W.F("migration: failed to encode compact event for serial %d: %v", m.Serial, encErr) + return nil // Continue with next event + } + + // Store compact event + ser := new(types.Uint40) + if err = ser.Set(m.Serial); chk.E(err) { + return nil // Continue with next event + } + cmpKey := new(bytes.Buffer) + if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); chk.E(err) { + return nil // Continue with next event + } + if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) { + log.W.F("migration: failed to store compact event for serial %d: %v", m.Serial, err) + return nil // Continue with next event + } + + // Track savings + savedBytes += int64(len(m.OldData) - len(compactData)) + processedCount++ + + // Cache the mappings + d.serialCache.CacheEventId(m.Serial, m.EventId) return nil }); chk.E(err) { - log.W.F("batch migration failed: %v", err) + log.W.F("migration failed for event %d: %v", m.Serial, err) continue } - if (i/batchSize)%10 == 0 && i > 0 { - log.I.F("migration progress: %d/%d events converted", i, len(migrations)) + // Log progress every 1000 events + if (i+1)%1000 == 0 { + log.I.F("migration progress: %d/%d events converted", i+1, len(migrations)) } } diff --git a/pkg/version/version b/pkg/version/version index 9d31093..5357063 100644 --- a/pkg/version/version +++ b/pkg/version/version @@ -1 +1 @@ -v0.34.3 \ No newline at end of file +v0.34.4 \ No newline at end of file