#!/bin/bash # Badger Database Migration Script # Migrates ORLY database to new Badger configuration with VLogPercentile optimization set -e # Exit on error # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color echo -e "${GREEN}=== ORLY Badger Database Migration ===${NC}" echo "" # Configuration DATA_DIR="${ORLY_DATA_DIR:-$HOME/.local/share/ORLY}" BACKUP_DIR="${DATA_DIR}-backup-$(date +%Y%m%d-%H%M%S)" EXPORT_FILE="${DATA_DIR}/events-export.jsonl" RELAY_BIN="${RELAY_BIN:-./orly}" # Check if relay binary exists if [ ! -f "$RELAY_BIN" ]; then echo -e "${RED}Error: ORLY binary not found at $RELAY_BIN${NC}" echo "Please build the relay first: go build -o orly" echo "Or set RELAY_BIN environment variable to the binary location" exit 1 fi # Check if database exists if [ ! -d "$DATA_DIR" ]; then echo -e "${YELLOW}Warning: Database directory not found at $DATA_DIR${NC}" echo "Nothing to migrate. If this is a fresh install, you can skip migration." exit 0 fi # Check disk space DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1) AVAILABLE_SPACE=$(df "$HOME" | tail -1 | awk '{print $4}') AVAILABLE_SPACE=$((AVAILABLE_SPACE * 1024)) # Convert to bytes REQUIRED_SPACE=$((DB_SIZE * 3)) # 3x for safety (export + backup + new DB) echo "Database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" echo "Available space: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)" echo "Required space: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)" echo "" if [ $AVAILABLE_SPACE -lt $REQUIRED_SPACE ]; then echo -e "${RED}Error: Not enough disk space!${NC}" echo "Required: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)" echo "Available: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)" echo "" echo "Options:" echo " 1. Free up disk space" echo " 2. Use natural compaction (no migration needed)" echo " 3. Export to external drive and import back" exit 1 fi # Check if relay is running if pgrep -x "orly" > /dev/null; then echo -e "${YELLOW}Warning: ORLY relay is currently running${NC}" echo "The relay should be stopped before migration." echo "" read -p "Stop the relay now? (y/N) " -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]]; then echo "Attempting to stop relay..." if systemctl is-active --quiet orly; then sudo systemctl stop orly echo -e "${GREEN}Relay stopped via systemd${NC}" else pkill orly sleep 2 if pgrep -x "orly" > /dev/null; then echo -e "${RED}Failed to stop relay. Please stop it manually and try again.${NC}" exit 1 fi echo -e "${GREEN}Relay stopped${NC}" fi else echo "Please stop the relay and run this script again." exit 1 fi fi echo "" echo -e "${YELLOW}=== Migration Plan ===${NC}" echo "1. Export all events to JSONL: $EXPORT_FILE" echo "2. Backup current database to: $BACKUP_DIR" echo "3. Create new database with optimized configuration" echo "4. Import all events (rebuilds indexes)" echo "5. Verify event counts match" echo "" echo "Estimated time: $(( (DB_SIZE / 1024 / 1024 / 100) + 1 )) - $(( (DB_SIZE / 1024 / 1024 / 50) + 1 )) minutes" echo "" read -p "Proceed with migration? (y/N) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then echo "Migration cancelled." exit 0 fi # Step 1: Export events echo "" echo -e "${GREEN}=== Step 1: Exporting Events ===${NC}" echo "This may take several minutes for large databases..." echo "" # We'll use a Go program to export since the binary doesn't have a CLI export command # Create temporary export program EXPORT_PROG=$(mktemp -d)/export-db.go cat > "$EXPORT_PROG" << 'EOF' package main import ( "context" "fmt" "os" "next.orly.dev/pkg/database" ) func main() { if len(os.Args) < 3 { fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) os.Exit(1) } dataDir := os.Args[1] outFile := os.Args[2] ctx := context.Background() cancel := func() {} db, err := database.New(ctx, cancel, dataDir, "error") if err != nil { fmt.Fprintf(os.Stderr, "Failed to open database: %v\n", err) os.Exit(1) } defer db.Close() f, err := os.Create(outFile) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create output file: %v\n", err) os.Exit(1) } defer f.Close() fmt.Println("Exporting events...") db.Export(ctx, f) fmt.Println("Export complete!") } EOF # Build and run export program echo "Building export tool..." EXPORT_BIN=$(mktemp) if ! go build -o "$EXPORT_BIN" "$EXPORT_PROG" 2>&1; then echo -e "${RED}Failed to build export tool${NC}" rm -f "$EXPORT_PROG" "$EXPORT_BIN" exit 1 fi echo "Running export..." if ! "$EXPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then echo -e "${RED}Export failed!${NC}" rm -f "$EXPORT_PROG" "$EXPORT_BIN" exit 1 fi rm -f "$EXPORT_PROG" "$EXPORT_BIN" # Count exported events EXPORT_COUNT=$(wc -l < "$EXPORT_FILE") echo -e "${GREEN}Exported $EXPORT_COUNT events${NC}" echo "Export size: $(du -h "$EXPORT_FILE" | cut -f1)" # Step 2: Backup current database echo "" echo -e "${GREEN}=== Step 2: Backing Up Current Database ===${NC}" echo "Moving $DATA_DIR to $BACKUP_DIR" mv "$DATA_DIR" "$BACKUP_DIR" echo -e "${GREEN}Backup complete${NC}" # Step 3 & 4: Create new database and import echo "" echo -e "${GREEN}=== Step 3 & 4: Creating New Database and Importing ===${NC}" echo "This will take longer as indexes are rebuilt..." echo "" # Create temporary import program IMPORT_PROG=$(mktemp -d)/import-db.go cat > "$IMPORT_PROG" << 'EOF' package main import ( "context" "fmt" "os" "next.orly.dev/pkg/database" ) func main() { if len(os.Args) < 3 { fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) os.Exit(1) } dataDir := os.Args[1] importFile := os.Args[2] ctx := context.Background() cancel := func() {} // This will create new database with updated configuration from database.go db, err := database.New(ctx, cancel, dataDir, "info") if err != nil { fmt.Fprintf(os.Stderr, "Failed to create database: %v\n", err) os.Exit(1) } defer db.Close() f, err := os.Open(importFile) if err != nil { fmt.Fprintf(os.Stderr, "Failed to open import file: %v\n", err) os.Exit(1) } defer f.Close() fmt.Println("Importing events (this may take a while)...") db.Import(f) // Wait for import to complete fmt.Println("Import started. Waiting for completion...") fmt.Println("Check the log output above for progress (logged every 100 events)") } EOF # Build and run import program echo "Building import tool..." IMPORT_BIN=$(mktemp) if ! go build -o "$IMPORT_BIN" "$IMPORT_PROG" 2>&1; then echo -e "${RED}Failed to build import tool${NC}" echo "Rolling back..." mv "$BACKUP_DIR" "$DATA_DIR" rm -f "$IMPORT_PROG" "$IMPORT_BIN" exit 1 fi echo "Running import..." if ! "$IMPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then echo -e "${RED}Import failed!${NC}" echo "Rolling back..." rm -rf "$DATA_DIR" mv "$BACKUP_DIR" "$DATA_DIR" rm -f "$IMPORT_PROG" "$IMPORT_BIN" exit 1 fi rm -f "$IMPORT_PROG" "$IMPORT_BIN" # Give import goroutine time to process echo "Waiting for import to complete..." sleep 10 # Step 5: Verify echo "" echo -e "${GREEN}=== Step 5: Verification ===${NC}" NEW_DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1) echo "Old database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" echo "New database size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)" echo "" if [ $NEW_DB_SIZE -lt $((DB_SIZE / 10)) ]; then echo -e "${YELLOW}Warning: New database is suspiciously small${NC}" echo "This may indicate an incomplete import." echo "Check the logs in $DATA_DIR/migration.log" echo "" read -p "Continue anyway? (y/N) " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then echo "Rolling back..." rm -rf "$DATA_DIR" mv "$BACKUP_DIR" "$DATA_DIR" exit 1 fi fi echo -e "${GREEN}=== Migration Complete! ===${NC}" echo "" echo "Summary:" echo " - Exported: $EXPORT_COUNT events" echo " - Old DB size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" echo " - New DB size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)" echo " - Space saved: $(numfmt --to=iec-i --suffix=B $((DB_SIZE - NEW_DB_SIZE)))" echo " - Backup location: $BACKUP_DIR" echo "" echo "Next steps:" echo " 1. Start the relay: sudo systemctl start orly (or ./orly)" echo " 2. Monitor performance for 24-48 hours" echo " 3. Watch for cache hit ratio >85% in logs" echo " 4. Verify event count and queries work correctly" echo " 5. After verification, remove backup: rm -rf $BACKUP_DIR" echo "" echo "Rollback (if needed):" echo " Stop relay, then: rm -rf $DATA_DIR && mv $BACKUP_DIR $DATA_DIR" echo ""