{"version": 3, "term": {"cols": 120, "rows": 42}, "timestamp": 1793680367, "idle_time_limit": 2.0, "title": "harvey: VM DR with NetApp SnapMirror"}
[0.043, "o", "\u001b[1;36m# Reprotect: reverse resync, DR becomes the source, the old primary the replica\u001b[0m\r\n"]
[1.011, "o", "\u001b[1;32m$\u001b[0m h"]
[0.033, "o", "a"]
[0.032, "o", "r"]
[0.033, "o", "v"]
[0.034, "o", "e"]
[0.030, "o", "y"]
[0.052, "o", " "]
[0.029, "o", "r"]
[0.029, "o", "e"]
[0.032, "o", "p"]
[0.029, "o", "r"]
[0.033, "o", "o"]
[0.032, "o", "t"]
[0.035, "o", "e"]
[0.035, "o", "c"]
[0.030, "o", "t"]
[0.033, "o", " "]
[0.030, "o", "-"]
[0.033, "o", "n"]
[0.032, "o", " "]
[0.031, "o", "h"]
[0.034, "o", "a"]
[0.035, "o", "r"]
[0.038, "o", "v"]
[0.037, "o", "e"]
[0.039, "o", "y"]
[0.036, "o", "-"]
[0.038, "o", "d"]
[0.041, "o", "r"]
[0.035, "o", "-"]
[0.037, "o", "d"]
[0.038, "o", "e"]
[0.037, "o", "m"]
[0.032, "o", "o"]
[0.032, "o", " "]
[0.037, "o", "d"]
[0.036, "o", "r"]
[0.031, "o", "-"]
[0.035, "o", "d"]
[0.035, "o", "e"]
[0.034, "o", "m"]
[0.035, "o", "o"]
[0.038, "o", "-"]
[0.039, "o", "1"]
[0.031, "o", " "]
[0.037, "o", "-"]
[0.035, "o", "-"]
[0.032, "o", "y"]
[0.035, "o", "e"]
[0.036, "o", "s"]
[0.036, "o", "\r\n"]
[61.660, "o", "  primary: tmr/dr-demo-\u001b[1;36m1\u001b[0m-root -> reestablished <- svm_dr:dr_pvc_305f2520_8fb4_4414_bbec_7db78a3e6a41\r\n"]
[0.000, "o", "  primary: tmr/dr-demo-\u001b[1;36m1\u001b[0m-data-iscsi -> reestablished <- svm_dr:dr_pvc_96975775_ebca_4350_97ac_38c8852283f0\r\n"]
[0.001, "o", "  primary: tmr/dr-demo-\u001b[1;36m1\u001b[0m-data-nfs -> reestablished <- svm_dr:dr_pvc_9ec9eb75_48c5_4412_9488_be6bf398a182\r\n"]
[0.000, "o", "  primary: secret/dr-demo-\u001b[1;36m1\u001b[0m-cloudinit updated\r\n"]
[0.000, "o", "  primary: nad/harvey-dr-demo/demo-vlan999 present\r\n"]
[0.000, "o", "  primary: vm/dr-demo-\u001b[1;36m1\u001b[0m updated \u001b[1m(\u001b[0mHalted\u001b[1m)\u001b[0m\r\n"]
[0.015, "o", "  \u001b[32m✓ harvey-dr-demo/dr-demo-\u001b[0m\u001b[1;32m1\u001b[0m\u001b[32m replicating dr → primary\u001b[0m\r\n"]
[0.063, "o", "\r\n"]
[21.532, "o", "\u001b[1;32m$\u001b[0m h"]
[0.032, "o", "a"]
[0.034, "o", "r"]
[0.040, "o", "v"]
[0.032, "o", "e"]
[0.039, "o", "y"]
[0.051, "o", " "]
[0.036, "o", "r"]
[0.032, "o", "e"]
[0.030, "o", "p"]
[0.030, "o", "l"]
[0.033, "o", "i"]
[0.038, "o", "c"]
[0.041, "o", "a"]
[0.041, "o", "t"]
[0.041, "o", "i"]
[0.039, "o", "o"]
[0.040, "o", "n"]
[0.044, "o", " "]
[0.037, "o", "s"]
[0.042, "o", "t"]
[0.036, "o", "a"]
[0.038, "o", "t"]
[0.036, "o", "u"]
[0.040, "o", "s"]
[0.033, "o", "\r\n"]
[1.575, "o", "\u001b[3m                                                   Replication status                                                   \u001b[0m\r\n╭──────────────────────────┬──────────────────────┬─────────────┬──────────────┬─────────┬─────┬────────────────┬──────╮\r\n│\u001b[1m \u001b[0m\u001b[1mVM / direction / state  \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mVolume              \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mTMR pri→rep\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mSnapMirror  \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mHealthy\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mLag\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mRecovery point\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mNote\u001b[0m\u001b[1m \u001b[0m│\r\n├──────────────────────────┼──────────────────────┼─────────────┼─"]
[0.000, "o", "─────────────┼─────────┼─────┼────────────────┼──────┤\r\n│ \u001b[1mharvey-dr-demo/dr-demo-1\u001b[0m │ dr-demo-1-root       │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 46s │ 04:33:29       │      │\r\n│ dr → primary             │ dr-demo-1-data-iscsi │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 15s │ 04:34:00       │      │\r\n│ Running / Stopped        │ dr-demo-1-data-nfs   │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 15s │ 04:34:00       │      │\r\n╰──────────────────────────┴──────────────────────┴─────────────┴──────────────┴─────────┴─────┴────────────────┴──────╯\r\n"]
[0.003, "o", "Worst lag: \u001b[1m46s\u001b[0m  \u001b[1m(\u001b[0mschedule \u001b[32m'harvey-1min'\u001b[0m; lag = age of the newest replicated snapshot = worst-case data loss if the \r\nprimary site fails now\u001b[1m)\u001b[0m\r\n"]
[0.028, "o", "\r\n"]
[1.516, "o", "\u001b[1;36m# Failback: planned failover back to the primary site, then reprotect in the original direction\u001b[0m\r\n"]
[1.017, "o", "\u001b[1;32m$\u001b[0m "]
[0.001, "o", "h"]
[0.040, "o", "a"]
[0.033, "o", "r"]
[0.037, "o", "v"]
[0.033, "o", "e"]
[0.038, "o", "y"]
[0.036, "o", " "]
[0.035, "o", "f"]
[0.036, "o", "a"]
[0.039, "o", "i"]
[0.037, "o", "l"]
[0.042, "o", "b"]
[0.044, "o", "a"]
[0.040, "o", "c"]
[0.039, "o", "k"]
[0.042, "o", " "]
[0.043, "o", "-"]
[0.031, "o", "n"]
[0.040, "o", " "]
[0.039, "o", "h"]
[0.036, "o", "a"]
[0.034, "o", "r"]
[0.034, "o", "v"]
[0.037, "o", "e"]
[0.039, "o", "y"]
[0.036, "o", "-"]
[0.035, "o", "d"]
[0.035, "o", "r"]
[0.037, "o", "-"]
[0.038, "o", "d"]
[0.036, "o", "e"]
[0.039, "o", "m"]
[0.033, "o", "o"]
[0.033, "o", " "]
[0.035, "o", "d"]
[0.038, "o", "r"]
[0.036, "o", "-"]
[0.034, "o", "d"]
[0.044, "o", "e"]
[0.037, "o", "m"]
[0.034, "o", "o"]
[0.039, "o", "-"]
[0.036, "o", "1"]
[0.036, "o", " "]
[0.032, "o", "-"]
[0.032, "o", "-"]
[0.031, "o", "y"]
[0.036, "o", "e"]
[0.035, "o", "s"]
[0.037, "o", "\r\n"]
[0.675, "o", "\u001b[?25l"]
[0.060, "o", "\u001b[32m⠋\u001b[0m \u001b[36mFetching VM inventory from both clusters...\u001b[0m\r\n\u001b[?25h\r\u001b[1A\u001b[2K"]
[0.000, "o", "\u001b[33m╭──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮\u001b[0m\r\n\u001b[33m│\u001b[0m Failback dr-demo-1: \u001b[32mdr\u001b[0m → \u001b[1mprimary\u001b[0m, then reprotect primary → dr                                                        \u001b[33m│\u001b[0m\r\n\u001b[33m╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\r\n"]
[0.001, "o", "\u001b[3m                        Failover Plan: \u001b[0m\u001b[3;36mdr\u001b[0m\u001b[3m → \u001b[0m\u001b[3;32mprimary\u001b[0m\u001b[3m                         \u001b[0m\r\n╭────────┬─────────┬────────────────┬───────────┬──────────┬───────────────╮\r\n│\u001b[1m \u001b[0m\u001b[1mStop #\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mStart #\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mNamespace     \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mVM Name  \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mPriority\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mStartup Delay\u001b[0m\u001b[1m \u001b[0m│\r\n├────────┼─────────┼────────────────┼───────────┼──────────┼───────────────┤\r\n│\u001b[33m \u001b[0m\u001b[33m     1\u001b[0m\u001b[33m \u001b[0m│\u001b[32m \u001b[0m\u001b[32m      1\u001b[0m\u001b[32m \u001b[0m│\u001b[36m \u001b[0m\u001b[36mharvey-dr-demo\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35mdr-demo-1\u001b[0m\u001b[35m \u001b[0m│      100 │             — │\r\n"]
[0.000, "o", "╰────────┴─────────┴────────────────┴───────────┴──────────┴───────────────╯\r\n"]
[0.002, "o", "\u001b[92m────────────────────────────────────────────────────── \u001b[0m\u001b[1;34mStop Phase\u001b[0m\u001b[92m ──────────────────────────────────────────────────────\u001b[0m\r\n"]
[0.001, "o", "  \u001b[33mStop\u001b[0m  harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m  \u001b[1m(\u001b[0mpriority \u001b[1;36m100\u001b[0m\u001b[1m)\u001b[0m\r\n"]
[0.000, "o", "\u001b[92m───────────────────────────────────────────────────── \u001b[0m\u001b[1;34mStart Phase\u001b[0m\u001b[92m ──────────────────────────────────────────────────────\u001b[0m\r\n"]
[0.000, "o", "  \u001b[32mStart\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m  \u001b[1m(\u001b[0mpriority \u001b[1;36m100\u001b[0m\u001b[1m)\u001b[0m\r\n"]
[0.000, "o", "\u001b[92m────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\r\n"]
[0.025, "o", "\u001b[1mPreflight\u001b[0m\r\n"]
[0.066, "o", "  harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m: \u001b[1;36m3\u001b[0m \u001b[1;35mvolume\u001b[0m\u001b[1m(\u001b[0ms\u001b[1m)\u001b[0m ready\r\n"]
[0.000, "o", "\u001b[1mMetadata sync\u001b[0m\r\n"]
[0.264, "o", "\u001b[1mStop on dr\u001b[0m\r\n"]
[6.219, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m stopped\r\n"]
[0.001, "o", "\u001b[1mFinal SnapMirror update\u001b[0m\r\n"]
[15.318, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m-root: \u001b[1;36m3661632\u001b[0m B in \u001b[1;36m15.\u001b[0m3s\r\n"]
[15.257, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m-data-iscsi: \u001b[1;36m92768\u001b[0m B in \u001b[1;36m15.\u001b[0m2s\r\n"]
[9.253, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m-data-nfs: \u001b[1;36m100000\u001b[0m B in \u001b[1;36m9.\u001b[0m2s\r\n"]
[0.000, "o", "\u001b[1mPromote on primary\u001b[0m\r\n"]
[34.161, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m: \u001b[1;36m3\u001b[0m \u001b[1;35mvolume\u001b[0m\u001b[1m(\u001b[0ms\u001b[1m)\u001b[0m read-write\r\n"]
[0.000, "o", "\u001b[1mStart on primary\u001b[0m\r\n"]
[15.377, "o", "  \u001b[32m✓\u001b[0m harvey-dr-demo/dr-demo-\u001b[1;36m1\u001b[0m Running\r\n"]
[0.006, "o", "\u001b[92m────────────────────────────────────────────────── \u001b[0m\u001b[1;32mFailover Complete\u001b[0m\u001b[92m ───────────────────────────────────────────────────\u001b[0m\r\n"]
[0.004, "o", "\u001b[3m                Failover Summary                \u001b[0m\r\n╭──────────────────────────────┬───────────────╮\r\n│\u001b[1m \u001b[0m\u001b[1mMetric                      \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1m        Value\u001b[0m\u001b[1m \u001b[0m│\r\n├──────────────────────────────┼───────────────┤\r\n│\u001b[1m \u001b[0m\u001b[1;34mVMs stopped on source\u001b[0m\u001b[1m       \u001b[0m\u001b[1m \u001b[0m│             1 │\r\n│\u001b[1m \u001b[0m\u001b[1;35mVolumes final-synced\u001b[0m\u001b[1m        \u001b[0m\u001b[1m \u001b[0m│             3 │\r\n│\u001b[1m \u001b[0m\u001b[1;32mVMs started on DR\u001b[0m\u001b[1m           \u001b[0m\u001b[1m \u001b[0m│             1 │\r\n│\u001b[1m \u001b[0m\u001b[1;2m  stop\u001b[0m\u001b[1m                      \u001b[0m\u001b[1m \u001b[0m│            6s │\r\n│\u001b[1m \u001b[0m\u001b[1;2m  final_sync\u001b[0m\u001b[1m                \u001b[0m\u001b[1m \u001b[0m│           39s │\r\n│\u001b[1m \u001b[0m\u001b[1;2m  promote\u001b[0m\u001b[1m                   \u001b[0m\u001b[1m \u001b[0m│           34s │\r\n│\u001b[1m \u001b[0m\u001b[1;2m  start\u001b["]
[0.000, "o", "0m\u001b[1m                     \u001b[0m\u001b[1m \u001b[0m│           15s │\r\n│\u001b[1m \u001b[0m\u001b[1;36mTotal RTO\u001b[0m\u001b[1m                   \u001b[0m\u001b[1m \u001b[0m│ 95s  (1m 35s) │\r\n│\u001b[1m \u001b[0m\u001b[1;36mRPO harvey-dr-demo/dr-demo-1\u001b[0m\u001b[1m \u001b[0m│ 0s (lossless) │\r\n╰──────────────────────────────┴───────────────╯\r\n"]
[15.676, "o", "  dr: tmr/dr-demo-\u001b[1;36m1\u001b[0m-root -> reestablished <- svm0:rs_pvc_83b5a8b8_777a_49d5_a752_55d4f915a69e\r\n"]
[0.001, "o", "  dr: tmr/dr-demo-\u001b[1;36m1\u001b[0m-data-iscsi -> reestablished <- svm0:rs_pvc_122223bd_2ef5_446c_b8f4_65fcb92bb4b1\r\n"]
[0.001, "o", "  dr: tmr/dr-demo-\u001b[1;36m1\u001b[0m-data-nfs -> reestablished <- svm0:rs_pvc_3d313080_b728_438a_9182_579a1cf45e7d\r\n"]
[0.000, "o", "  dr: secret/dr-demo-\u001b[1;36m1\u001b[0m-cloudinit updated\r\n"]
[0.000, "o", "  dr: nad/harvey-dr-demo/demo-vlan999 present\r\n"]
[0.001, "o", "  dr: vm/dr-demo-\u001b[1;36m1\u001b[0m updated \u001b[1m(\u001b[0mHalted\u001b[1m)\u001b[0m\r\n"]
[0.093, "o", "\r\n"]
[1.520, "o", "\u001b[1;32m$\u001b[0m "]
[0.000, "o", "v"]
[0.039, "o", "i"]
[0.038, "o", "r"]
[0.033, "o", "t"]
[0.036, "o", "c"]
[0.035, "o", "t"]
[0.033, "o", "l"]
[0.033, "o", " "]
[0.041, "o", "s"]
[0.041, "o", "s"]
[0.038, "o", "h"]
[0.040, "o", " "]
[0.041, "o", "s"]
[0.035, "o", "l"]
[0.036, "o", "e"]
[0.033, "o", "s"]
[0.035, "o", "@"]
[0.035, "o", "v"]
[0.033, "o", "m"]
[0.036, "o", "/"]
[0.031, "o", "d"]
[0.034, "o", "r"]
[0.037, "o", "-"]
[0.034, "o", "d"]
[0.039, "o", "e"]
[0.037, "o", "m"]
[0.032, "o", "o"]
[0.038, "o", "-"]
[0.040, "o", "1"]
[0.038, "o", " "]
[0.042, "o", "("]
[0.042, "o", "p"]
[0.043, "o", "r"]
[0.037, "o", "i"]
[0.031, "o", "m"]
[0.042, "o", "a"]
[0.026, "o", "r"]
[0.033, "o", "y"]
[0.030, "o", ")"]
[0.033, "o", " "]
[0.032, "o", "-"]
[0.035, "o", "-"]
[0.034, "o", " "]
[0.033, "o", "c"]
[0.032, "o", "h"]
[0.033, "o", "e"]
[0.037, "o", "c"]
[0.037, "o", "k"]
[0.037, "o", " "]
[0.031, "o", "m"]
[0.031, "o", "a"]
[0.036, "o", "r"]
[0.034, "o", "k"]
[0.031, "o", "e"]
[0.030, "o", "r"]
[0.032, "o", " "]
[0.040, "o", "f"]
[0.038, "o", "i"]
[0.031, "o", "l"]
[0.039, "o", "e"]
[0.037, "o", "s"]
[0.033, "o", " "]
[0.040, "o", "f"]
[0.029, "o", "o"]
[0.031, "o", "r"]
[0.032, "o", " "]
[0.033, "o", "g"]
[0.033, "o", "a"]
[0.032, "o", "p"]
[0.034, "o", "s"]
[0.036, "o", "\r\n"]
[123.504, "o", "== /var/lib/harvey-marker: 468 lines, last 2026-11-03T04:38:17Z\r\n   gap 2026-11-03T04:34:18Z -> 2026-11-03T04:36:36Z\r\n== /mnt/iscsi/marker: 479 lines, last 2026-11-03T04:38:17Z\r\n   gap 2026-11-03T04:34:18Z -> 2026-11-03T04:36:36Z\r\n== /mnt/nfs/marker: 479 lines, last 2026-11-03T04:38:17Z\r\n   gap 2026-11-03T04:34:18Z -> 2026-11-03T04:36:36Z\r\n\r\n"]
[22.038, "o", "\u001b[1;32m$\u001b[0m "]
[0.000, "o", "h"]
[0.040, "o", "a"]
[0.039, "o", "r"]
[0.038, "o", "v"]
[0.038, "o", "e"]
[0.037, "o", "y"]
[0.031, "o", " "]
[0.033, "o", "r"]
[0.032, "o", "e"]
[0.038, "o", "p"]
[0.033, "o", "l"]
[0.034, "o", "i"]
[0.034, "o", "c"]
[0.033, "o", "a"]
[0.035, "o", "t"]
[0.035, "o", "i"]
[0.035, "o", "o"]
[0.031, "o", "n"]
[0.037, "o", " "]
[0.036, "o", "s"]
[0.035, "o", "t"]
[0.036, "o", "a"]
[0.037, "o", "t"]
[0.034, "o", "u"]
[0.035, "o", "s"]
[0.034, "o", "\r\n"]
[1.636, "o", "\u001b[3m                                                   Replication status                                                   \u001b[0m\r\n╭──────────────────────────┬──────────────────────┬─────────────┬──────────────┬─────────┬─────┬────────────────┬──────╮\r\n│\u001b[1m \u001b[0m\u001b[1mVM / direction / state  \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mVolume              \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mTMR pri→rep\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mSnapMirror  \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mHealthy\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mLag\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mRecovery point\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mNote\u001b[0m\u001b[1m \u001b[0m│\r\n├──────────────────────────┼──────────────────────┼─────────────┼─"]
[0.000, "o", "─────────────┼─────────┼─────┼────────────────┼──────┤\r\n│ \u001b[1mharvey-dr-demo/dr-demo-1\u001b[0m │ dr-demo-1-root       │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 42s │ 04:38:00       │      │\r\n│ primary → dr             │ dr-demo-1-data-iscsi │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 44s │ 04:38:00       │      │\r\n│ Running / Stopped        │ dr-demo-1-data-nfs   │ prom→re-est │ snapmirrored │ \u001b[32myes\u001b[0m     │ 43s │ 04:38:00       │      │\r\n╰──────────────────────────┴──────────────────────┴─────────────┴──────────────┴─────────┴─────┴────────────────┴──────╯\r\n"]
[0.003, "o", "Worst lag: \u001b[1m44s\u001b[0m  \u001b[1m(\u001b[0mschedule \u001b[32m'harvey-1min'\u001b[0m; lag = age of the newest replicated snapshot = worst-case data loss if the \r\nprimary site fails now\u001b[1m)\u001b[0m\r\n"]
[0.025, "o", "\r\n"]
[1.520, "x", "0"]