lein test jepsen.system.etcd-test INFO jepsen.os.debian - :n2 setting up debian INFO jepsen.os.debian - :n3 setting up debian INFO jepsen.os.debian - :n4 setting up debian INFO jepsen.os.debian - :n1 setting up debian INFO jepsen.os.debian - :n5 setting up debian INFO jepsen.os.debian - :n1 debian set up INFO jepsen.os.debian - :n4 debian set up INFO jepsen.os.debian - :n2 debian set up INFO jepsen.os.debian - :n3 debian set up INFO jepsen.os.debian - :n5 debian set up INFO jepsen.system.etcd - :n3 etcd nuked INFO jepsen.system.etcd - :n4 etcd nuked INFO jepsen.system.etcd - :n5 etcd nuked INFO jepsen.system.etcd - :n1 etcd nuked INFO jepsen.system.etcd - :n2 etcd nuked INFO jepsen.system.etcd - Running nodes: {:n1 false, :n2 false, :n3 false, :n4 false, :n5 false} INFO jepsen.system.etcd - :n2 etcd nuked INFO jepsen.system.etcd - :n4 etcd nuked INFO jepsen.system.etcd - :n5 etcd nuked INFO jepsen.system.etcd - :n3 etcd nuked INFO jepsen.system.etcd - :n1 etcd nuked INFO jepsen.system.etcd - :n1 starting etcd INFO jepsen.system.etcd - :n2 starting etcd INFO jepsen.system.etcd - :n5 starting etcd INFO jepsen.system.etcd - :n3 starting etcd INFO jepsen.system.etcd - :n4 starting etcd INFO jepsen.system.etcd - Running nodes: {:n1 true, :n2 true, :n3 true, :n4 true, :n5 true} INFO jepsen.system.etcd - :n1 etcd ready INFO jepsen.system.etcd - :n4 etcd ready INFO jepsen.system.etcd - :n5 etcd ready INFO jepsen.system.etcd - :n2 etcd ready INFO jepsen.system.etcd - :n3 etcd ready INFO jepsen.core - Worker 2 starting INFO jepsen.core - Worker 1 starting INFO jepsen.core - Worker 0 starting INFO jepsen.core - Worker 3 starting INFO jepsen.core - Worker 4 starting INFO jepsen.util - 0 :invoke :read nil INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 1 :invoke :read nil INFO jepsen.util - 2 :ok :read nil INFO jepsen.util - 0 :ok :read nil INFO jepsen.util - 1 :ok :read nil INFO jepsen.util - 3 :invoke :write 1 INFO jepsen.util - 4 :invoke :cas [3 3] INFO jepsen.util - 3 :ok :write 1 INFO jepsen.util - 4 :fail :cas [3 3] INFO jepsen.util - 2 :invoke :cas [2 0] INFO jepsen.util - 0 :invoke :write 0 INFO jepsen.util - 1 :invoke :read nil INFO jepsen.util - 1 :ok :read 1 INFO jepsen.util - 0 :ok :write 0 INFO jepsen.util - 3 :invoke :cas [3 0] INFO jepsen.util - 2 :fail :cas [2 0] INFO jepsen.util - 3 :fail :cas [3 0] INFO jepsen.util - 4 :invoke :cas [0 2] INFO jepsen.util - 4 :ok :cas [0 2] INFO jepsen.util - 1 :invoke :cas [3 4] INFO jepsen.util - 0 :invoke :cas [2 2] INFO jepsen.util - 1 :fail :cas [3 4] INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 3 :invoke :write 4 INFO jepsen.util - 2 :ok :read 2 INFO jepsen.util - 0 :ok :cas [2 2] INFO jepsen.util - 3 :ok :write 4 INFO jepsen.util - 4 :invoke :read nil INFO jepsen.util - 4 :ok :read 4 INFO jepsen.util - 1 :invoke :cas [0 1] INFO jepsen.util - 1 :fail :cas [0 1] INFO jepsen.util - 2 :invoke :cas [3 2] INFO jepsen.util - 0 :invoke :cas [3 3] INFO jepsen.util - 2 :fail :cas [3 2] INFO jepsen.util - 3 :invoke :read nil INFO jepsen.util - 0 :fail :cas [3 3] INFO jepsen.util - 3 :ok :read 4 INFO jepsen.util - 4 :invoke :write 4 INFO jepsen.util - 4 :ok :write 4 INFO jepsen.util - :nemesis :info :start nil INFO jepsen.util - 1 :invoke :cas [2 0] INFO jepsen.util - 1 :fail :cas [2 0] INFO jepsen.util - 2 :invoke :cas [3 1] INFO jepsen.util - 0 :invoke :cas [1 2] INFO jepsen.util - 3 :invoke :cas [2 0] INFO jepsen.util - 4 :invoke :write 0 INFO jepsen.util - 0 :fail :cas [1 2] INFO jepsen.util - 2 :fail :cas [3 1] INFO jepsen.util - 4 :ok :write 0 INFO jepsen.util - :nemesis :info :start "Cut off {:n1 #{:n4 :n2}, :n5 #{:n4 :n2}, :n3 #{:n4 :n2}, :n2 #{:n3 :n5 :n1}, :n4 #{:n3 :n5 :n1}}" INFO jepsen.util - 1 :invoke :write 2 INFO jepsen.util - 3 :info :cas :timed-out INFO jepsen.util - 0 :invoke :cas [2 2] INFO jepsen.util - 0 :fail :cas [2 2] INFO jepsen.util - 2 :invoke :write 4 INFO jepsen.util - 4 :invoke :read nil INFO jepsen.util - 4 :ok :read 0 INFO jepsen.util - 2 :ok :write 4 INFO jepsen.util - 1 :info :write :timed-out INFO jepsen.util - 8 :invoke :cas [2 0] INFO jepsen.util - 0 :invoke :write 3 INFO jepsen.util - 0 :ok :write 3 INFO jepsen.util - 4 :invoke :read nil INFO jepsen.util - 4 :ok :read 3 INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 2 :ok :read 3 INFO jepsen.util - 6 :invoke :write 3 INFO jepsen.util - 8 :info :cas :timed-out INFO jepsen.util - 0 :invoke :read nil INFO jepsen.util - 0 :ok :read 3 INFO jepsen.util - 4 :invoke :cas [2 2] INFO jepsen.util - 4 :fail :cas [2 2] INFO jepsen.util - 2 :invoke :cas [3 1] INFO jepsen.util - 6 :info :write :timed-out INFO jepsen.util - 13 :invoke :write 3 INFO jepsen.util - 2 :ok :cas [3 1] INFO jepsen.util - 0 :invoke :read nil INFO jepsen.util - 0 :ok :read 1 INFO jepsen.util - 4 :invoke :write 4 INFO jepsen.util - 11 :invoke :cas [1 4] INFO jepsen.util - 13 :info :write :timed-out INFO jepsen.util - 4 :ok :write 4 INFO jepsen.util - 2 :invoke :cas [2 4] INFO jepsen.util - 2 :fail :cas [2 4] INFO jepsen.util - 0 :invoke :read nil INFO jepsen.util - 0 :ok :read 4 INFO jepsen.util - :nemesis :info :stop nil INFO jepsen.util - :nemesis :info :stop "fully connected" INFO jepsen.util - 11 :info :cas :timed-out INFO jepsen.util - 18 :invoke :write 1 INFO jepsen.util - 4 :invoke :read nil INFO jepsen.util - 4 :ok :read 4 INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 2 :ok :read 4 INFO jepsen.util - 0 :invoke :write 1 INFO jepsen.util - 16 :invoke :cas [4 2] INFO jepsen.util - 18 :info :write :timed-out INFO jepsen.util - 4 :invoke :cas [3 4] INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 2 :ok :read 4 INFO jepsen.util - 0 :info :write :timed-out INFO jepsen.util - 16 :info :cas :timed-out INFO jepsen.util - 23 :invoke :write 1 INFO jepsen.util - 4 :info :cas :timed-out INFO jepsen.util - 2 :invoke :write 2 INFO jepsen.util - 5 :invoke :write 2 INFO jepsen.util - 2 :ok :write 2 INFO jepsen.util - 23 :ok :write 1 INFO jepsen.util - 5 :ok :write 2 INFO jepsen.util - 21 :invoke :write 1 INFO jepsen.util - 9 :invoke :read nil INFO jepsen.util - 21 :ok :write 1 INFO jepsen.util - 9 :ok :read 1 INFO jepsen.util - 23 :invoke :read nil INFO jepsen.util - 2 :invoke :cas [1 2] INFO jepsen.util - 5 :invoke :read nil INFO jepsen.util - 23 :ok :read 1 INFO jepsen.util - 5 :ok :read 1 INFO jepsen.util - 2 :ok :cas [1 2] INFO jepsen.util - 21 :invoke :write 4 INFO jepsen.util - 9 :invoke :read nil INFO jepsen.util - 9 :ok :read 2 INFO jepsen.util - 21 :ok :write 4 INFO jepsen.util - 23 :invoke :read nil INFO jepsen.util - 5 :invoke :read nil INFO jepsen.util - 23 :ok :read 4 INFO jepsen.util - 5 :ok :read 4 INFO jepsen.util - 2 :invoke :read nil INFO jepsen.util - 2 :ok :read 4 INFO jepsen.util - :nemesis :info :start nil INFO jepsen.util - 9 :invoke :write 3 INFO jepsen.util - 9 :ok :write 3 INFO jepsen.util - 21 :invoke :cas [2 4] INFO jepsen.util - 23 :invoke :write 0 INFO jepsen.util - 5 :invoke :read nil INFO jepsen.util - 5 :ok :read 3 INFO jepsen.util - 2 :invoke :cas [4 2] INFO jepsen.util - :nemesis :info :start "Cut off {:n1 #{:n2 :n5}, :n4 #{:n2 :n5}, :n3 #{:n2 :n5}, :n5 #{:n3 :n4 :n1}, :n2 #{:n3 :n4 :n1}}" INFO jepsen.util - 9 :invoke :cas [4 4] INFO jepsen.util - 21 :info :cas :timed-out INFO jepsen.util - 23 :info :write :timed-out INFO jepsen.util - 5 :invoke :write 0 INFO jepsen.util - 2 :info :cas :timed-out INFO jepsen.util - 9 :info :cas :timed-out INFO jepsen.util - 26 :invoke :write 2 INFO jepsen.util - 28 :invoke :cas [2 4] INFO jepsen.util - 5 :info :write :timed-out INFO jepsen.util - 28 :fail :cas [2 4] INFO jepsen.util - 7 :invoke :write 4 INFO jepsen.util - 7 :ok :write 4 INFO jepsen.util - 14 :invoke :write 2 INFO jepsen.util - 26 :info :write :timed-out INFO jepsen.util - 10 :invoke :cas [1 3] INFO jepsen.util - 28 :invoke :cas [0 1] INFO jepsen.util - 7 :invoke :cas [3 4] INFO jepsen.util - 10 :fail :cas [1 3] INFO jepsen.util - 28 :fail :cas [0 1] INFO jepsen.util - 7 :fail :cas [3 4] INFO jepsen.util - 14 :info :write :timed-out INFO jepsen.util - 31 :invoke :read nil INFO jepsen.util - 31 :ok :read 3 INFO jepsen.util - 10 :invoke :read nil INFO jepsen.util - 10 :ok :read 4 INFO jepsen.util - 28 :invoke :cas [0 0] INFO jepsen.util - 28 :fail :cas [0 0] INFO jepsen.util - :nemesis :info :stop nil INFO jepsen.util - :nemesis :info :stop "fully connected" INFO jepsen.util - :nemesis :info :stop nil INFO jepsen.util - :nemesis :info :stop "fully connected" INFO jepsen.core - nemesis done INFO jepsen.util - 7 :invoke :read nil INFO jepsen.core - Worker 0 done INFO jepsen.core - Worker 4 done INFO jepsen.core - Worker 1 done INFO jepsen.core - Worker 3 done INFO jepsen.util - 7 :ok :read 4 INFO jepsen.core - Worker 2 done INFO jepsen.core - Run complete, writing INFO jepsen.core - Analyzing INFO knossos.core - [160 / 172] 5038 extant worlds, 6819 visited/s, 603 skipped/s, hitrate 0.081 cache size 10709 INFO knossos.core - [160 / 172] 15835 extant worlds, 14134 visited/s, 1189 skipped/s, hitrate 0.078 cache size 30461 INFO knossos.core - [160 / 172] 23813 extant worlds, 11762 visited/s, 1033 skipped/s, hitrate 0.081 cache size 45355 INFO knossos.core - [160 / 172] 30733 extant worlds, 12215 visited/s, 1174 skipped/s, hitrate 0.088 cache size 59398 INFO knossos.core - [160 / 172] 39927 extant worlds, 14631 visited/s, 1366 skipped/s, hitrate 0.085 cache size 75610 INFO knossos.core - [160 / 172] 48009 extant worlds, 13823 visited/s, 1329 skipped/s, hitrate 0.088 cache size 90453 INFO knossos.core - [160 / 172] 52123 extant worlds, 9970 visited/s, 1310 skipped/s, hitrate 0.116 cache size 99568 INFO knossos.core - [160 / 172] 57987 extant worlds, 11065 visited/s, 996 skipped/s, hitrate 0.083 cache size 111659 INFO knossos.core - [160 / 172] 64560 extant worlds, 8487 visited/s, 759 skipped/s, hitrate 0.082 cache size 122769 INFO knossos.core - [160 / 172] 70383 extant worlds, 9078 visited/s, 739 skipped/s, hitrate 0.075 cache size 133551 INFO knossos.core - [160 / 172] 74813 extant worlds, 6844 visited/s, 609 skipped/s, hitrate 0.082 cache size 141583 INFO knossos.core - [160 / 172] 81958 extant worlds, 10427 visited/s, 875 skipped/s, hitrate 0.077 cache size 153588 INFO knossos.core - [160 / 172] 83606 extant worlds, 4291 visited/s, 628 skipped/s, hitrate 0.128 cache size 157579 INFO knossos.core - [160 / 172] 84500 extant worlds, 2556 visited/s, 350 skipped/s, hitrate 0.121 cache size 159512 INFO knossos.core - [160 / 172] 86485 extant worlds, 5742 visited/s, 715 skipped/s, hitrate 0.111 cache size 164771 INFO knossos.core - [160 / 172] 87426 extant worlds, 3178 visited/s, 463 skipped/s, hitrate 0.127 cache size 166999 INFO knossos.core - [160 / 172] 90226 extant worlds, 5563 visited/s, 580 skipped/s, hitrate 0.095 cache size 172958 INFO knossos.core - [160 / 172] 92997 extant worlds, 5300 visited/s, 568 skipped/s, hitrate 0.097 cache size 178592 INFO knossos.core - [160 / 172] 94570 extant worlds, 4000 visited/s, 530 skipped/s, hitrate 0.117 cache size 181987 INFO knossos.core - [160 / 172] 96207 extant worlds, 3042 visited/s, 325 skipped/s, hitrate 0.097 cache size 185419 INFO knossos.core - [160 / 172] 96487 extant worlds, 776 visited/s, 134 skipped/s, hitrate 0.148 cache size 185939 INFO knossos.core - [160 / 172] 99640 extant worlds, 5563 visited/s, 628 skipped/s, hitrate 0.102 cache size 190964 INFO knossos.core - [160 / 172] 102477 extant worlds, 6854 visited/s, 915 skipped/s, hitrate 0.118 cache size 196689 INFO knossos.core - [160 / 172] 106251 extant worlds, 6819 visited/s, 775 skipped/s, hitrate 0.102 cache size 204400 INFO knossos.core - [160 / 172] 111600 extant worlds, 9634 visited/s, 929 skipped/s, hitrate 0.088 cache size 215481 INFO knossos.core - [160 / 172] 117057 extant worlds, 9501 visited/s, 907 skipped/s, hitrate 0.087 cache size 225385 INFO knossos.core - [160 / 172] 120399 extant worlds, 5127 visited/s, 518 skipped/s, hitrate 0.092 cache size 230853 INFO knossos.core - [160 / 172] 123799 extant worlds, 7967 visited/s, 1008 skipped/s, hitrate 0.112 cache size 238115 INFO knossos.core - [160 / 172] 129591 extant worlds, 10632 visited/s, 1060 skipped/s, hitrate 0.091 cache size 249742 INFO knossos.core - [160 / 172] 134978 extant worlds, 11168 visited/s, 1147 skipped/s, hitrate 0.093 cache size 260732 INFO knossos.core - [160 / 172] 138080 extant worlds, 8107 visited/s, 1074 skipped/s, hitrate 0.117 cache size 267445 INFO knossos.core - [160 / 172] 142127 extant worlds, 7791 visited/s, 870 skipped/s, hitrate 0.100 cache size 275345 INFO knossos.core - [160 / 172] 143296 extant worlds, 2681 visited/s, 389 skipped/s, hitrate 0.127 cache size 277580 INFO knossos.core - [160 / 172] 145803 extant worlds, 4269 visited/s, 469 skipped/s, hitrate 0.099 cache size 282346 INFO knossos.core - [160 / 172] 148837 extant worlds, 5085 visited/s, 410 skipped/s, hitrate 0.075 cache size 286976 INFO knossos.core - [160 / 172] 149950 extant worlds, 3225 visited/s, 543 skipped/s, hitrate 0.144 cache size 288874 INFO knossos.core - [160 / 172] 152934 extant worlds, 5939 visited/s, 608 skipped/s, hitrate 0.093 cache size 295531 INFO knossos.core - [160 / 172] 153855 extant worlds, 4137 visited/s, 593 skipped/s, hitrate 0.126 cache size 299075 INFO knossos.core - [160 / 172] 155879 extant worlds, 5596 visited/s, 792 skipped/s, hitrate 0.124 cache size 303951 INFO knossos.core - [160 / 172] 160592 extant worlds, 10046 visited/s, 1122 skipped/s, hitrate 0.101 cache size 314487 INFO knossos.core - [160 / 172] 165197 extant worlds, 6725 visited/s, 762 skipped/s, hitrate 0.102 cache size 322414 INFO knossos.core - [160 / 172] 168475 extant worlds, 7964 visited/s, 939 skipped/s, hitrate 0.105 cache size 329894 INFO knossos.core - [160 / 172] 170746 extant worlds, 5327 visited/s, 742 skipped/s, hitrate 0.122 cache size 334601 INFO knossos.core - [160 / 172] 173930 extant worlds, 6828 visited/s, 906 skipped/s, hitrate 0.117 cache size 340788 INFO knossos.core - [160 / 172] 176899 extant worlds, 5997 visited/s, 756 skipped/s, hitrate 0.112 cache size 346994 INFO knossos.core - [160 / 172] 180619 extant worlds, 8396 visited/s, 1276 skipped/s, hitrate 0.132 cache size 354329 INFO knossos.core - [160 / 172] 180580 extant worlds, 3800 visited/s, 743 skipped/s, hitrate 0.164 cache size 356095 INFO knossos.core - [160 / 172] 180581 extant worlds, 2293 visited/s, 544 skipped/s, hitrate 0.192 cache size 356794 INFO knossos.core - [160 / 172] 181085 extant worlds, 2264 visited/s, 463 skipped/s, hitrate 0.170 cache size 358000 INFO knossos.core - [160 / 172] 184227 extant worlds, 5775 visited/s, 640 skipped/s, hitrate 0.100 cache size 364322 INFO knossos.core - [160 / 172] 187829 extant worlds, 5978 visited/s, 605 skipped/s, hitrate 0.092 cache size 370758 INFO knossos.core - [160 / 172] 190282 extant worlds, 4267 visited/s, 347 skipped/s, hitrate 0.075 cache size 375554 INFO knossos.core - [160 / 172] 193120 extant worlds, 4291 visited/s, 425 skipped/s, hitrate 0.090 cache size 381084 INFO knossos.core - [160 / 172] 193862 extant worlds, 3916 visited/s, 718 skipped/s, hitrate 0.155 cache size 383885 INFO knossos.core - [160 / 172] 195345 extant worlds, 6035 visited/s, 970 skipped/s, hitrate 0.139 cache size 388089 INFO knossos.core - [160 / 172] 195339 extant worlds, 2197 visited/s, 449 skipped/s, hitrate 0.170 cache size 388904 INFO knossos.core - [160 / 172] 195342 extant worlds, 2173 visited/s, 509 skipped/s, hitrate 0.190 cache size 389709 INFO knossos.core - [160 / 172] 195348 extant worlds, 2552 visited/s, 516 skipped/s, hitrate 0.168 cache size 390822 INFO knossos.core - [160 / 172] 195337 extant worlds, 1484 visited/s, 458 skipped/s, hitrate 0.236 cache size 391266 INFO knossos.core - [160 / 172] 195343 extant worlds, 631 visited/s, 140 skipped/s, hitrate 0.182 cache size 391412 INFO knossos.core - [160 / 172] 195328 extant worlds, 815 visited/s, 203 skipped/s, hitrate 0.200 cache size 391663 INFO knossos.core - [160 / 172] 195326 extant worlds, 1628 visited/s, 397 skipped/s, hitrate 0.196 cache size 391983 INFO knossos.core - [160 / 172] 195324 extant worlds, 1542 visited/s, 309 skipped/s, hitrate 0.167 cache size 392595 INFO knossos.core - [160 / 172] 195342 extant worlds, 975 visited/s, 195 skipped/s, hitrate 0.167 cache size 392911 INFO knossos.core - [160 / 172] 195340 extant worlds, 1257 visited/s, 248 skipped/s, hitrate 0.165 cache size 393382 INFO knossos.core - [160 / 172] 195319 extant worlds, 2965 visited/s, 499 skipped/s, hitrate 0.144 cache size 394288 INFO knossos.core - [160 / 172] 195307 extant worlds, 2392 visited/s, 470 skipped/s, hitrate 0.164 cache size 395254 INFO knossos.core - [160 / 172] 195302 extant worlds, 1140 visited/s, 218 skipped/s, hitrate 0.161 cache size 395540 INFO knossos.core - [160 / 172] 195318 extant worlds, 943 visited/s, 222 skipped/s, hitrate 0.191 cache size 395849 INFO knossos.core - [160 / 172] 195348 extant worlds, 295 visited/s, 210 skipped/s, hitrate 0.416 cache size 395888 INFO knossos.core - [160 / 172] 195342 extant worlds, 62 visited/s, 13 skipped/s, hitrate 0.180 cache size 395904 INFO knossos.core - [160 / 172] 195356 extant worlds, 38 visited/s, 17 skipped/s, hitrate 0.312 cache size 395919 TIMEOUT after 2 days