Fix OOM in BuildIndexes by processing in chunks (v0.48.6)

- Process events in 200k chunks instead of loading all at once
- Write indexes to disk after each chunk, then free memory
- Call debug.FreeOSMemory() between chunks to release memory to OS
- Memory usage now ~150-200MB per chunk instead of 5GB+

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
woikos
2026-01-06 09:10:50 +01:00
parent d363f5da04
commit 2480be3a73
5 changed files with 355 additions and 10 deletions

View File

@@ -127,14 +127,21 @@ func (b *B) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
continue
}
log.D.F("bbolt import: policy allowed event %x during sync import", ev.ID)
}
if _, err := b.SaveEvent(ctx, ev); err != nil {
// return the pooled buffer on error paths too
ev.Free()
saveErrors++
log.W.F("bbolt import: failed to save event: %v", err)
continue
// With policy checking, use regular SaveEvent path
if _, err := b.SaveEvent(ctx, ev); err != nil {
ev.Free()
saveErrors++
log.W.F("bbolt import: failed to save event: %v", err)
continue
}
} else {
// Minimal path for migration: store events only, build indexes later
if err := b.SaveEventMinimal(ev); err != nil {
ev.Free()
saveErrors++
log.W.F("bbolt import: failed to save event: %v", err)
continue
}
}
// return the pooled buffer after successful save
@@ -174,6 +181,15 @@ func (b *B) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
return err
}
// Build indexes after minimal import (when no policy manager = migration mode)
if policyManager == nil && count > 0 {
log.I.F("bbolt import: building indexes for %d events...", count)
if err := b.BuildIndexes(ctx); err != nil {
log.E.F("bbolt import: failed to build indexes: %v", err)
return err
}
}
return nil
}