JUnit version 4.13.2
.23:08:18.171 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783465698163
23:08:18.232 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:08:18.267 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:08:18.279 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskcoHO5i/test-tmp
23:08:18.304 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:08:18.307 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskcoHO5i/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:08:18.525 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:08:18.730 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:08:18.731 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:08:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:18.735100 24433 mini_oidc.cc:150] Starting JWKS server
23:08:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:18.739379 24433 webserver.cc:534] Webserver started at http://127.0.0.1:35943/ using document root <none> and password file <none>
23:08:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:18.741115 24433 mini_oidc.cc:186] Starting OIDC Discovery server
23:08:18.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:18.745082 24433 webserver.cc:534] Webserver started at http://127.0.0.1:41003/ using document root <none> and password file <none>
23:08:18.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:08:18Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
23:08:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:08:18Z Disabled control of system clock
23:08:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:18.820140 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal
23:08:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/logs
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data/info.pb
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:08:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.126:38141
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.126
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json with env {}
23:08:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.133903 24445 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.134536 24445 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:19.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.134997 24445 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.170925 24445 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:08:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.171239 24445 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.171450 24445 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:08:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.171639 24445 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:08:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.171823 24445 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:08:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.171962 24445 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35943/jwks.json
23:08:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:19.209446 24445 master_runner.cc:387] Master server non-default flags:
23:08:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data
23:08:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.126:38141
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json
23:08:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data/info.pb
23:08:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.126
23:08:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/logs
23:08:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:08:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:19.210958 24445 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:19.212857 24445 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.225178 24451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:21.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:20.629582 24450 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 24445
23:08:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.081669 24450 kernel_stack_watchdog.cc:198] Thread 24445 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 401ms:
23:08:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:08:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:08:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:08:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:08:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:19.225972 24452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.082932 24445 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.857s	user 0.697s	sys 1.156s
23:08:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.083477 24445 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.858s	user 0.697s	sys 1.157s
23:08:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.086087 24454 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.090649 24453 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1862 milliseconds
23:08:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.090653 24445 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.092126 24445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.094727 24445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.096145 24445 hybrid_clock.cc:648] HybridClock initialized: now 1783465701096109 us; error 70 us; skew 500 ppm
23:08:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.097028 24445 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.103456 24445 webserver.cc:534] Webserver started at http://127.23.220.126:36687/ using document root <none> and password file <none>
23:08:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.104488 24445 fs_manager.cc:362] Metadata directory not provided
23:08:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.104688 24445 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.105142 24445 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.109692 24445 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data/instance:
23:08:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c7573fee0045e6b4cf6b74d506a57f"
23:08:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:21 on dist-test-slave-dr0q"
23:08:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.110975 24445 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal/instance:
23:08:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c7573fee0045e6b4cf6b74d506a57f"
23:08:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:21 on dist-test-slave-dr0q"
23:08:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.118949 24445 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.002s
23:08:21.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.125195 24461 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.126868 24445 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
23:08:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.127322 24445 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal
23:08:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c7573fee0045e6b4cf6b74d506a57f"
23:08:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:21 on dist-test-slave-dr0q"
23:08:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.127830 24445 fs_report.cc:389] FS layout report
23:08:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data/data
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.189549 24445 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:21.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.191108 24445 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:21.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.191588 24445 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:21.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.266245 24445 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.126:38141
23:08:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.266350 24512 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.126:38141 every 8 connection(s)
23:08:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.269169 24445 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/data/info.pb
23:08:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.270998 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 24445
23:08:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.271621 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-0/wal/instance
23:08:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.275990 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal
23:08:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data
23:08:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:21.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/logs
23:08:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data/info.pb
23:08:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:21.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.125:38335
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.125
23:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:21.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:21.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json with env {}
23:08:21.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.276539 24513 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.290794 24513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } has no permanent_uuid. Determining permanent_uuid...
23:08:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.319427 24513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } has no permanent_uuid. Determining permanent_uuid...
23:08:21.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.322212 24462 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:21.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.327323 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.376977 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 1
23:08:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.381824 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.450371 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 2
23:08:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.455008 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.546530 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 3
23:08:21.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.551348 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.602921 24515 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.603562 24515 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.604310 24515 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.646061 24515 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:08:21.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.646368 24515 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.646584 24515 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:08:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.646772 24515 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:08:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.646955 24515 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:08:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.647091 24515 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35943/jwks.json
23:08:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.683928 24515 master_runner.cc:387] Master server non-default flags:
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.125:38335
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data/info.pb
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.125
23:08:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/logs
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.685261 24515 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.687541 24515 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:21.694809 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 4
23:08:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.701884 24524 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.701781 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:22.001462 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 5
23:08:22.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:22.017792 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:22.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:22.565403 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 6
23:08:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:22.592312 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.547190 24515 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.845s	user 0.705s	sys 1.135s
23:08:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.105468 24523 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 24515
23:08:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:21.702899 24525 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.548317 24515 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.846s	user 0.705s	sys 1.135s
23:08:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.549188 24526 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1845 milliseconds
23:08:23.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.551254 24527 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.551262 24515 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.554888 24515 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.557626 24515 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.559039 24515 hybrid_clock.cc:648] HybridClock initialized: now 1783465703559002 us; error 47 us; skew 500 ppm
23:08:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.560050 24515 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.566964 24515 webserver.cc:534] Webserver started at http://127.23.220.125:46819/ using document root <none> and password file <none>
23:08:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.568176 24515 fs_manager.cc:362] Metadata directory not provided
23:08:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.568456 24515 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.569092 24515 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:23.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.573922 24515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data/instance:
23:08:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbc90b2b56554d558ecdf7692710afb0"
23:08:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:23 on dist-test-slave-dr0q"
23:08:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.575536 24515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal/instance:
23:08:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbc90b2b56554d558ecdf7692710afb0"
23:08:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:23 on dist-test-slave-dr0q"
23:08:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.583834 24515 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.006s
23:08:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.590096 24536 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:23.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.591392 24515 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
23:08:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.591708 24515 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal
23:08:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbc90b2b56554d558ecdf7692710afb0"
23:08:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:23 on dist-test-slave-dr0q"
23:08:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.592038 24515 fs_report.cc:389] FS layout report
23:08:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal
23:08:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal
23:08:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data/data
23:08:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:23.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.652910 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 7
23:08:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.661232 24513 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.125:38335: Network error: Client connection negotiation failed: client connection to 127.23.220.125:38335: connect: Connection refused (error 111)
23:08:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.669819 24515 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.671446 24515 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.671904 24515 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.746734 24515 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.125:38335
23:08:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.746836 24589 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.125:38335 every 8 connection(s)
23:08:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.749763 24515 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/data/info.pb
23:08:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.756105 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 24515
23:08:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.756574 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-1/wal/instance
23:08:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.756930 24590 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.758947 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/logs
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data/info.pb
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.124:40087
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.124
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json with env {}
23:08:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.770092 24590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } has no permanent_uuid. Determining permanent_uuid...
23:08:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.798010 24590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } has no permanent_uuid. Determining permanent_uuid...
23:08:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.815665 24590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
23:08:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.818383 24537 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.823356 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.872994 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 1
23:08:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.878008 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:23.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:23.946455 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 2
23:08:23.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:23.951140 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:24.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.042677 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 3
23:08:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.047435 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.090742 24592 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.091388 24592 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.091899 24592 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.129331 24592 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.129635 24592 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.129832 24592 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.130023 24592 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.130220 24592 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.130362 24592 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35943/jwks.json
23:08:24.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.168730 24592 master_runner.cc:387] Master server non-default flags:
23:08:24.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data
23:08:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal
23:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.124:40087
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35943/jwks.json
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data/info.pb
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.124
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/logs
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.170186 24592 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.172178 24592 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:24.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.184598 24602 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.190893 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 4
23:08:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.195429 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:24.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:24.494854 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 5
23:08:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.499538 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.046924 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 6
23:08:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.056988 24590 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.220.124:40087: Network error: Client connection negotiation failed: client connection to 127.23.220.124:40087: connect: Connection refused (error 111)
23:08:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:24.186444 24603 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.391541 24604 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:08:25.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.393044 24605 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:25.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.393059 24592 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.396339 24592 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.399667 24592 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.401177 24592 hybrid_clock.cc:648] HybridClock initialized: now 1783465705401128 us; error 74 us; skew 500 ppm
23:08:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.402107 24592 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.409157 24592 webserver.cc:534] Webserver started at http://127.23.220.124:42027/ using document root <none> and password file <none>
23:08:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.410101 24592 fs_manager.cc:362] Metadata directory not provided
23:08:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.410288 24592 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.410773 24592 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:25.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.415273 24592 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data/instance:
23:08:25.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4ab4f192d26490c923c7359453f4f0d"
23:08:25.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:25 on dist-test-slave-dr0q"
23:08:25.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.416385 24592 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal/instance:
23:08:25.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4ab4f192d26490c923c7359453f4f0d"
23:08:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:25 on dist-test-slave-dr0q"
23:08:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.426101 24592 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.010s	sys 0.000s
23:08:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.432067 24614 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.433531 24592 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.433853 24592 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4ab4f192d26490c923c7359453f4f0d"
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:25 on dist-test-slave-dr0q"
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.434171 24592 fs_report.cc:389] FS layout report
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal
23:08:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data/data
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.491246 24592 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.492817 24592 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.493258 24592 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:25.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.567349 24592 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.124:40087
23:08:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.567492 24665 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.124:40087 every 8 connection(s)
23:08:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.570458 24592 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/data/info.pb
23:08:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.576584 24666 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.580857 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 24592
23:08:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.581243 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/master-2/wal/instance
23:08:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.589077 24666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } has no permanent_uuid. Determining permanent_uuid...
23:08:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.618746 24666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } has no permanent_uuid. Determining permanent_uuid...
23:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.629915 24666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
23:08:25.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.647574 24666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: Bootstrap starting.
23:08:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.654605 24666 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: Neither blocks nor log segments found. Creating new log.
23:08:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.656528 24666 log.cc:826] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: Log is configured to *not* fsync() on all Append() calls
23:08:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.662665 24666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: No bootstrap required, opened a new log
23:08:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.682809 24666 raft_consensus.cc:359] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.684450 24666 raft_consensus.cc:385] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.685145 24666 raft_consensus.cc:740] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4ab4f192d26490c923c7359453f4f0d, State: Initialized, Role: FOLLOWER
23:08:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.686209 24666 consensus_queue.cc:260] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.688719 24675 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.689359 24675 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: This master's current role is: FOLLOWER
23:08:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.690964 24666 sys_catalog.cc:565] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: configured and running, proceeding with master startup.
23:08:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.725939 24686 catalog_manager.cc:1593] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:08:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.726343 24686 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:08:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.721424 24592 master_runner.cc:187] Error getting master registration for 127.23.220.126:38141: OK, instance_id { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" instance_seqno: 1783465701245548 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:08:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.743742 24592 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:08:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.748541 24620 tablet.cc:2406] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:08:25.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.751703 24513 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } attempt: 8
23:08:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.778822 24513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } has no permanent_uuid. Determining permanent_uuid...
23:08:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.801088 24513 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: Bootstrap starting.
23:08:25.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.809563 24513 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: Neither blocks nor log segments found. Creating new log.
23:08:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.812109 24513 log.cc:826] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: Log is configured to *not* fsync() on all Append() calls
23:08:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.817847 24513 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: No bootstrap required, opened a new log
23:08:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.839263 24513 raft_consensus.cc:359] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.840096 24513 raft_consensus.cc:385] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.840404 24513 raft_consensus.cc:740] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 39c7573fee0045e6b4cf6b74d506a57f, State: Initialized, Role: FOLLOWER
23:08:25.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.841392 24513 consensus_queue.cc:260] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.845175 24690 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:25.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.846071 24690 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: This master's current role is: FOLLOWER
23:08:25.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.847245 24513 sys_catalog.cc:565] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: configured and running, proceeding with master startup.
23:08:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.883003 24701 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:08:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.884061 24701 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:08:25.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.881915 24445 master_runner.cc:187] Error getting master registration for 127.23.220.125:38335: OK, instance_id { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" instance_seqno: 1783465703725618 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:08:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.917649 24445 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:08:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.918541 24675 raft_consensus.cc:493] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:08:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.919132 24675 raft_consensus.cc:515] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.923364 24675 leader_election.cc:290] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 39c7573fee0045e6b4cf6b74d506a57f (127.23.220.126:38141), cbc90b2b56554d558ecdf7692710afb0 (127.23.220.125:38335)
23:08:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.923470 24488 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4ab4f192d26490c923c7359453f4f0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "39c7573fee0045e6b4cf6b74d506a57f" is_pre_election: true
23:08:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.924474 24488 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a4ab4f192d26490c923c7359453f4f0d in term 0.
23:08:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.926049 24618 leader_election.cc:304] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39c7573fee0045e6b4cf6b74d506a57f, a4ab4f192d26490c923c7359453f4f0d; no voters: 
23:08:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.927150 24675 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Leader pre-election won for term 1
23:08:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.927521 24675 raft_consensus.cc:493] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:08:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.927852 24675 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 0 FOLLOWER]: Advancing to term 1
23:08:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.936100 24675 raft_consensus.cc:515] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.939512 24488 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4ab4f192d26490c923c7359453f4f0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "39c7573fee0045e6b4cf6b74d506a57f"
23:08:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.940176 24488 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 0 FOLLOWER]: Advancing to term 1
23:08:25.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:25.952272 24467 tablet.cc:2406] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:08:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.957914 24488 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a4ab4f192d26490c923c7359453f4f0d in term 1.
23:08:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.965628 24675 leader_election.cc:290] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [CANDIDATE]: Term 1 election: Requested vote from peers 39c7573fee0045e6b4cf6b74d506a57f (127.23.220.126:38141), cbc90b2b56554d558ecdf7692710afb0 (127.23.220.125:38335)
23:08:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.967404 24618 leader_election.cc:304] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39c7573fee0045e6b4cf6b74d506a57f, a4ab4f192d26490c923c7359453f4f0d; no voters: 
23:08:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.968315 24675 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 1 FOLLOWER]: Leader election won for term 1
23:08:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.970888 24675 raft_consensus.cc:697] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [term 1 LEADER]: Becoming Leader. State: Replica: a4ab4f192d26490c923c7359453f4f0d, State: Running, Role: LEADER
23:08:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.972347 24675 consensus_queue.cc:237] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.984087 24703 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4ab4f192d26490c923c7359453f4f0d. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.986330 24703 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: This master's current role is: LEADER
23:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:25.998764 24705 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
23:08:26.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.007584 24705 catalog_manager.cc:1519] Initializing Kudu cluster ID...
23:08:26.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.040406 24488 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [term 1 FOLLOWER]: Refusing update from remote peer a4ab4f192d26490c923c7359453f4f0d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:08:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.042268 24703 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [LEADER]: Connected to new peer: Peer: permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
23:08:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.074602 24690 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4ab4f192d26490c923c7359453f4f0d. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.075500 24690 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: This master's current role is: FOLLOWER
23:08:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.084816 24706 mvcc.cc:204] Tried to move back new op lower bound from 7305075531919450112 to 7305075531674075136. Current Snapshot: MvccSnapshot[applied={T|T < 7305075531919450112}]
23:08:26.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.088668 24703 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.089489 24703 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: This master's current role is: LEADER
23:08:26.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.102728 24690 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.106623 24705 catalog_manager.cc:1382] Generated new cluster ID: 5feedaebecf048468836dd3ec82010e8
23:08:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.107836 24705 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
23:08:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.117591 24590 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } attempt: 7
23:08:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.117926 24690 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f [sys.catalog]: This master's current role is: FOLLOWER
23:08:26.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.170847 24590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: Bootstrap starting.
23:08:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.171260 24705 catalog_manager.cc:1405] Generated new certificate authority record
23:08:26.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.175477 24705 catalog_manager.cc:1539] Loading token signing keys...
23:08:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.206068 24705 catalog_manager.cc:6069] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d: Generated new TSK 0
23:08:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.207412 24705 catalog_manager.cc:1549] Initializing in-progress tserver states...
23:08:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.208416 24590 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: Neither blocks nor log segments found. Creating new log.
23:08:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.213881 24590 log.cc:826] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: Log is configured to *not* fsync() on all Append() calls
23:08:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.233124 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:0
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:08:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.235103 24590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: No bootstrap required, opened a new log
23:08:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.304966 24590 raft_consensus.cc:359] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.307106 24590 raft_consensus.cc:385] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.307978 24590 raft_consensus.cc:740] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cbc90b2b56554d558ecdf7692710afb0, State: Initialized, Role: FOLLOWER
23:08:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.310473 24590 consensus_queue.cc:260] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } }
23:08:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.331332 24714 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.332343 24714 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: This master's current role is: FOLLOWER
23:08:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.337646 24563 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [term 0 FOLLOWER]: Advancing to term 1
23:08:26.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.342795 24564 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4ab4f192d26490c923c7359453f4f0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cbc90b2b56554d558ecdf7692710afb0"
23:08:26.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.344420 24565 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4ab4f192d26490c923c7359453f4f0d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cbc90b2b56554d558ecdf7692710afb0" is_pre_election: true
23:08:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.347657 24563 raft_consensus.cc:1275] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [term 1 FOLLOWER]: Refusing update from remote peer a4ab4f192d26490c923c7359453f4f0d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:08:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.348917 24590 sys_catalog.cc:565] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: configured and running, proceeding with master startup.
23:08:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.349556 24703 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [LEADER]: Connected to new peer: Peer: permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:08:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.449785 24714 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4ab4f192d26490c923c7359453f4f0d. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.449949 24719 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.451237 24714 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: This master's current role is: FOLLOWER
23:08:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.451261 24719 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0 [sys.catalog]: This master's current role is: FOLLOWER
23:08:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.484038 24730 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:08:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.754961 24703 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a4ab4f192d26490c923c7359453f4f0d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "39c7573fee0045e6b4cf6b74d506a57f" member_type: VOTER last_known_addr { host: "127.23.220.126" port: 38141 } } peers { permanent_uuid: "cbc90b2b56554d558ecdf7692710afb0" member_type: VOTER last_known_addr { host: "127.23.220.125" port: 38335 } } peers { permanent_uuid: "a4ab4f192d26490c923c7359453f4f0d" member_type: VOTER last_known_addr { host: "127.23.220.124" port: 40087 } } }
23:08:26.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.756285 24703 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4ab4f192d26490c923c7359453f4f0d [sys.catalog]: This master's current role is: LEADER
23:08:26.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.788633 24713 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:26.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.789928 24713 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:26.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.791042 24713 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:26.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.852706 24713 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.855211 24713 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.65
23:08:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.890798 24701 catalog_manager.cc:1294] Loaded cluster ID: 5feedaebecf048468836dd3ec82010e8
23:08:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.891111 24701 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: loading cluster ID for follower catalog manager: success
23:08:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.899322 24701 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: acquiring CA information for follower catalog manager: success
23:08:26.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.906751 24701 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 39c7573fee0045e6b4cf6b74d506a57f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:08:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.924060 24713 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:0
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.925951 24713 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:26.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:26.928306 24713 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.948122 24738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:27.497961 24730 catalog_manager.cc:1294] Loaded cluster ID: 5feedaebecf048468836dd3ec82010e8
23:08:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:27.498317 24730 catalog_manager.cc:1587] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: loading cluster ID for follower catalog manager: success
23:08:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:27.524675 24730 catalog_manager.cc:1609] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: acquiring CA information for follower catalog manager: success
23:08:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:27.547328 24730 catalog_manager.cc:1637] T 00000000000000000000000000000000 P cbc90b2b56554d558ecdf7692710afb0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:08:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:28.351063 24737 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 24713
23:08:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:28.688628 24713 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.743s	user 0.498s	sys 0.725s
23:08:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:28.699376 24713 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.754s	user 0.499s	sys 0.725s
23:08:28.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.707742 24713 server_base.cc:1061] running on GCE node
23:08:28.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:26.950711 24739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:28.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:28.715878 24745 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.718087 24713 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.727166 24713 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:28.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.729375 24713 hybrid_clock.cc:648] HybridClock initialized: now 1783465708729251 us; error 59 us; skew 500 ppm
23:08:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.731105 24713 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:28.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.752125 24713 webserver.cc:534] Webserver started at http://127.23.220.65:36877/ using document root <none> and password file <none>
23:08:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.754222 24713 fs_manager.cc:362] Metadata directory not provided
23:08:28.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.754987 24713 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:28.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.756011 24713 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.764185 24713 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/instance:
23:08:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:28.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:08:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.766678 24713 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal/instance:
23:08:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:08:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.784997 24713 fs_manager.cc:696] Time spent creating directory manager: real 0.017s	user 0.010s	sys 0.005s
23:08:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.800480 24750 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.803236 24713 fs_manager.cc:730] Time spent opening block manager: real 0.012s	user 0.003s	sys 0.005s
23:08:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.804445 24713 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:08:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.805869 24713 fs_report.cc:389] FS layout report
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/data
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.926481 24713 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.929148 24713 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.930181 24713 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.940198 24713 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.953613 24713 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:08:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.954398 24713 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.001s	sys 0.000s
23:08:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.955013 24713 ts_tablet_manager.cc:616] Registered 0 tablets
23:08:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:28.955454 24713 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:08:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.649871 24713 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.65:35727
23:08:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.649993 24863 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.65:35727 every 8 connection(s)
23:08:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.654856 24713 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:29.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.663838 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 24713
23:08:29.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.664702 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal/instance
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.696549 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:0
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.785883 24865 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.786448 24865 heartbeater.cc:461] Registering TS with master...
23:08:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.787919 24865 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.791396 24631 ts_manager.cc:194] Registered new tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.794360 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.65:57819
23:08:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.801170 24864 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.801575 24864 heartbeater.cc:461] Registering TS with master...
23:08:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.802279 24864 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:29.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.806499 24555 ts_manager.cc:194] Registered new tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:29.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.811064 24868 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:29.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.811498 24868 heartbeater.cc:461] Registering TS with master...
23:08:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.812150 24868 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:29.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:29.815508 24478 ts_manager.cc:194] Registered new tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.145848 24869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.146342 24869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.146852 24869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.180533 24869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.181772 24869 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.66
23:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:30.220949 24869 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:0
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:30.222405 24869 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:30.225298 24869 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.242489 24881 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:30.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:30.799820 24865 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:31.649900 24880 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 24869
23:08:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:31.951912 24869 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.708s	user 0.525s	sys 1.051s
23:08:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:31.952519 24869 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.709s	user 0.525s	sys 1.051s
23:08:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:30.244853 24882 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:31.953498 24883 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1708 milliseconds
23:08:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.954843 24869 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:31.954905 24884 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.960153 24869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:31.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.962451 24869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:31.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.963927 24869 hybrid_clock.cc:648] HybridClock initialized: now 1783465711963867 us; error 68 us; skew 500 ppm
23:08:31.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.964845 24869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.971432 24869 webserver.cc:534] Webserver started at http://127.23.220.66:34525/ using document root <none> and password file <none>
23:08:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.972433 24869 fs_manager.cc:362] Metadata directory not provided
23:08:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.972692 24869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.973191 24869 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.977630 24869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/instance:
23:08:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:08:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.978938 24869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal/instance:
23:08:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:08:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.986779 24869 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.000s
23:08:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.992652 24893 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.993911 24869 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
23:08:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.994340 24869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:31.994824 24869 fs_report.cc:389] FS layout report
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/data
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.052465 24869 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:32.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.054651 24869 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:32.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.055104 24869 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.057753 24869 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.062397 24869 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:08:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.062686 24869 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
23:08:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.062901 24869 ts_tablet_manager.cc:616] Registered 0 tablets
23:08:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.063033 24869 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:08:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.226969 24869 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.66:42405
23:08:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.227078 25005 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.66:42405 every 8 connection(s)
23:08:32.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.229775 24869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.232199 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 24869
23:08:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.232784 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal/instance
23:08:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.248934 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:08:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:0
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.273011 25006 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.273542 25006 heartbeater.cc:461] Registering TS with master...
23:08:32.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.275023 25006 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.277153 25008 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.277491 25008 heartbeater.cc:461] Registering TS with master...
23:08:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.277881 25007 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.278668 25007 heartbeater.cc:461] Registering TS with master...
23:08:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.278751 25008 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.279531 25007 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.279526 24555 ts_manager.cc:194] Registered new tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.281761 24631 ts_manager.cc:194] Registered new tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.282858 24478 ts_manager.cc:194] Registered new tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.283474 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.66:50945
23:08:32.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.613737 25016 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:32.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.614243 25016 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:32.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.614760 25016 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:32.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.647446 25016 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.648516 25016 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.67
23:08:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.686215 25016 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:0
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.688153 25016 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:32.690327 25016 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.705303 25022 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:33.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:33.287528 25007 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:32.707334 25023 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:34.023988 25025 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:34.035512 25024 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1315 milliseconds
23:08:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.035722 25016 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.037403 25016 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.043547 25016 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.045104 25016 hybrid_clock.cc:648] HybridClock initialized: now 1783465714045010 us; error 101 us; skew 500 ppm
23:08:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.046895 25016 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.061342 25016 webserver.cc:534] Webserver started at http://127.23.220.67:46209/ using document root <none> and password file <none>
23:08:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.063603 25016 fs_manager.cc:362] Metadata directory not provided
23:08:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.064697 25016 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.069099 25016 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:08:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.079044 25016 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/instance:
23:08:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:08:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.081702 25016 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal/instance:
23:08:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:08:34.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.093966 25016 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.009s	sys 0.000s
23:08:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.100577 25034 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.102311 25016 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.002s
23:08:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.102835 25016 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:08:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.103281 25016 fs_report.cc:389] FS layout report
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/data
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:34.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.166844 25016 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:34.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.168612 25016 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.169163 25016 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:34.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.172112 25016 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.177352 25016 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:08:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.177623 25016 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:08:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.177911 25016 ts_tablet_manager.cc:616] Registered 0 tablets
23:08:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.178171 25016 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:08:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.343904 25016 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.67:38085
23:08:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.344034 25146 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.67:38085 every 8 connection(s)
23:08:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.346836 25016 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.353294 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 25016
23:08:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.353842 24433 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal/instance
23:08:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.412789 25149 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.413297 25148 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.413407 25149 heartbeater.cc:461] Registering TS with master...
23:08:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.413683 25148 heartbeater.cc:461] Registering TS with master...
23:08:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.414635 25147 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.414994 25147 heartbeater.cc:461] Registering TS with master...
23:08:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.415156 25148 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.415087 25149 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.416042 25147 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.419219 24631 ts_manager.cc:194] Registered new tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.419534 24478 ts_manager.cc:194] Registered new tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.419615 24555 ts_manager.cc:194] Registered new tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.421787 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.67:60669
23:08:34.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:34.435899 24433 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
23:08:34.449 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:08:34.450 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:08:34.462 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.220.126"
      port: 38141
    }
    bound_http_address {
      host: "127.23.220.126"
      port: 36687
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.220.125"
      port: 38335
    }
    bound_http_address {
      host: "127.23.220.125"
      port: 46819
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.220.124"
      port: 40087
    }
    bound_http_address {
      host: "127.23.220.124"
      port: 42027
    }
  }
}

23:08:34.466 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:08:34.469 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.23.220.65"
      port: 35727
    }
    bound_http_address {
      host: "127.23.220.65"
      port: 36877
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.23.220.66"
      port: 42405
    }
    bound_http_address {
      host: "127.23.220.66"
      port: 34525
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.23.220.67"
      port: 38085
    }
    bound_http_address {
      host: "127.23.220.67"
      port: 46209
    }
  }
}

23:08:34.469 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:08:34.659 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
23:08:34.719 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:08:34.919 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.220.124:40087(127.23.220.124:40087)] connecting to peer
23:08:34.974 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.220.126:38141(127.23.220.126:38141)] connecting to peer
23:08:34.976 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.220.125:38335(127.23.220.125:38335)] connecting to peer
23:08:34.995 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.220.126:38141(127.23.220.126:38141)] Successfully connected to peer
23:08:34.995 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.23.220.125:38335(127.23.220.125:38335)] Successfully connected to peer
23:08:35.001 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.220.124:40087(127.23.220.124:40087)] Successfully connected to peer
23:08:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:35.067036 24509 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7cc0 after lost signal to thread 24446
23:08:35.108 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.220.124
23:08:35.115 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.220.126
23:08:35.121 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.220.125
23:08:35.404 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x85163878, L:/127.0.0.1:41750 - R:/127.23.220.126:38141] using SASL/PLAIN
23:08:35.407 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb0773e0a, L:/127.0.0.1:35128 - R:/127.23.220.124:40087] using SASL/PLAIN
23:08:35.407 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xf4b87cd6, L:/127.0.0.1:38998 - R:/127.23.220.125:38335] using SASL/PLAIN
23:08:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.425921 25148 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:35.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:08:35.459 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=587), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.220.124:40087, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.220.126:38141, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.220.125:38335, [548ms] Sub RPC ConnectToMaster: received response from server master-127.23.220.125:38335: OK, [548ms] Sub RPC ConnectToMaster: received response from server master-127.23.220.124:40087: OK, [558ms] Sub RPC ConnectToMaster: received response from server master-127.23.220.126:38141: OK, deferred=null) after lookup
23:08:35.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.516623 24631 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35128:
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853"
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:08:35.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:08:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:08:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:08:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:35.520757 24631 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:08:35.645 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table fc4ae3dcd4fc443bbe0e7ff93e214db5
23:08:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.692250 25082 tablet_service.cc:1511] Processing CreateTablet for tablet 50caf47eb7d047cf914475445d7f87e2 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853 [id=fc4ae3dcd4fc443bbe0e7ff93e214db5]), partition=RANGE (key) PARTITION UNBOUNDED
23:08:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.695039 25082 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50caf47eb7d047cf914475445d7f87e2. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.703250 24799 tablet_service.cc:1511] Processing CreateTablet for tablet 50caf47eb7d047cf914475445d7f87e2 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853 [id=fc4ae3dcd4fc443bbe0e7ff93e214db5]), partition=RANGE (key) PARTITION UNBOUNDED
23:08:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.705176 24799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50caf47eb7d047cf914475445d7f87e2. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.714433 24941 tablet_service.cc:1511] Processing CreateTablet for tablet 50caf47eb7d047cf914475445d7f87e2 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853 [id=fc4ae3dcd4fc443bbe0e7ff93e214db5]), partition=RANGE (key) PARTITION UNBOUNDED
23:08:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.716457 24941 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50caf47eb7d047cf914475445d7f87e2. 1 dirs total, 0 dirs full, 0 dirs failed
23:08:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.745478 25183 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap starting.
23:08:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.747388 25184 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap starting.
23:08:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.756374 25185 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap starting.
23:08:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.765090 25184 tablet_bootstrap.cc:654] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Neither blocks nor log segments found. Creating new log.
23:08:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.765093 25185 tablet_bootstrap.cc:654] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Neither blocks nor log segments found. Creating new log.
23:08:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.767333 25183 tablet_bootstrap.cc:654] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Neither blocks nor log segments found. Creating new log.
23:08:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.767578 25185 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Log is configured to *not* fsync() on all Append() calls
23:08:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.772564 25184 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Log is configured to *not* fsync() on all Append() calls
23:08:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.773335 25183 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Log is configured to *not* fsync() on all Append() calls
23:08:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.789072 25183 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: No bootstrap required, opened a new log
23:08:35.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.790719 25183 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent bootstrapping tablet: real 0.050s	user 0.013s	sys 0.007s
23:08:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.794397 25185 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: No bootstrap required, opened a new log
23:08:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.794974 25185 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent bootstrapping tablet: real 0.041s	user 0.026s	sys 0.012s
23:08:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.806951 25184 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: No bootstrap required, opened a new log
23:08:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.808022 25184 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent bootstrapping tablet: real 0.061s	user 0.015s	sys 0.003s
23:08:35.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.826023 25185 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.826990 25185 raft_consensus.cc:385] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:35.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.827310 25185 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Initialized, Role: FOLLOWER
23:08:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.829208 25185 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.839622 25185 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent starting tablet: real 0.044s	user 0.033s	sys 0.008s
23:08:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.860366 25183 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.861305 25183 raft_consensus.cc:385] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.861622 25183 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Initialized, Role: FOLLOWER
23:08:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.862739 25183 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.864730 25184 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.865816 25184 raft_consensus.cc:385] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:08:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.866236 25184 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Initialized, Role: FOLLOWER
23:08:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.867522 25184 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.873723 25184 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent starting tablet: real 0.065s	user 0.034s	sys 0.004s
23:08:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:35.874415 25183 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent starting tablet: real 0.083s	user 0.035s	sys 0.008s
23:08:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:35.886328 25152 tablet.cc:2406] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:08:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:35.993503 24872 tablet.cc:2406] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:08:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:36.012542 25012 tablet.cc:2406] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:08:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.069149 25189 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:08:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.069789 25189 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.073068 25189 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:36.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.085564 24819 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:08:36.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.086541 24819 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 0.
23:08:36.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.088313 24896 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 
23:08:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.089509 25102 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:08:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.089702 25189 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:08:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.090167 25189 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:08:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.090499 25102 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 0.
23:08:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.090876 25189 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 0 FOLLOWER]: Advancing to term 1
23:08:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.099694 25189 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.102296 25189 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 1 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.106853 25102 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.107417 25102 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 0 FOLLOWER]: Advancing to term 1
23:08:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.107769 24819 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.108352 24819 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 0 FOLLOWER]: Advancing to term 1
23:08:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.114957 25102 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 1.
23:08:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.115533 24819 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 1.
23:08:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.116255 24895 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 
23:08:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.117806 25189 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Leader election won for term 1
23:08:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.120154 25189 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 LEADER]: Becoming Leader. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Running, Role: LEADER
23:08:36.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.122247 25189 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.139736 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 reported cstate change: term changed from 0 to 1, leader changed from <none> to 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66). New cstate: current_term: 1 leader_uuid: "59052f49be5346a4ba48421e0b75dc49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:08:36.435 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table fc4ae3dcd4fc443bbe0e7ff93e214db5
23:08:36.456 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:36.461 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:36.463 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=null) after lookup
23:08:36.473 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:36.476 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] Successfully connected to peer
23:08:36.585 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x08a7abb8, L:/127.0.0.1:37092 - R:/127.23.220.66:42405] using TOKEN/null
23:08:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.612191 24819 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Refusing update from remote peer 59052f49be5346a4ba48421e0b75dc49: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:08:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.614394 25189 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
23:08:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.623404 25102 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Refusing update from remote peer 59052f49be5346a4ba48421e0b75dc49: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:08:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.625605 25189 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:08:36.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.736325 25212 mvcc.cc:204] Tried to move back new op lower bound from 7305075575218962432 to 7305075573247504384. Current Snapshot: MvccSnapshot[applied={T|T < 7305075575218962432}]
23:08:36.955 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:08:36.957 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.66:42405
23:08:36.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:08:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:36.959506 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 24869
23:08:37.007 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] handling channelInactive
23:08:37.008 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state READY due to: connection closed
23:08:37.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:37.024 [DEBUG - Time-limited test] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.028 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.029 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.029 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.039 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.039 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test)) after lookup
23:08:37.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.048 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.049 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.051 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.080 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.080 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449))) after lookup
23:08:37.081 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.084 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.084 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.084 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.138 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.139 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=115), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [57ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [60ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [60ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [108ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [113ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668))) after lookup
23:08:37.140 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.143 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.143 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.143 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.239 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.239 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.240 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=216), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [57ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [60ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [60ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [108ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [113ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [116ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [119ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [119ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [215ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805))) after lookup
23:08:37.240 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.242 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.439 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=416), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [57ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [60ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [60ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [108ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [113ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [116ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [119ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [119ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [215ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [216ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [218ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [218ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [409ms] refreshing cache from master, [409ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [414ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518))) after lookup
23:08:37.441 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.443 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.443 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:37.858 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:37.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=834), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [57ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [60ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [60ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [108ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [113ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [116ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [119ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [119ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [215ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [216ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [218ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [218ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [409ms] refreshing cache from master, [409ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [414ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [417ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [419ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [419ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [828ms] refreshing cache from master, [829ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [833ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518) -> (continuation of Deferred@2108342237 after retry RPC@1584877535), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518) -> (continuation of Deferred@2108342237 after retry RPC@1584877535))) after lookup
23:08:37.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:37.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:37.861 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.381673 25228 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:08:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.382161 25228 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:38.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.384958 25228 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.389183 25036 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.394089 25036 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.395349 25233 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:08:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.396112 25233 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:38.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.400215 25233 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.399967 24818 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.401643 24818 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 1.
23:08:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.406625 25037 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:08:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.407505 25228 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Leader pre-election won for term 2
23:08:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.407864 25228 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Starting leader election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:08:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.407866 24752 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.408218 25228 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 1 FOLLOWER]: Advancing to term 2
23:08:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.414489 24752 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.415490 25102 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:08:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.416105 25228 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.417081 25102 raft_consensus.cc:2393] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 641de1a253394bf595b9bc1c4ede893a in current term 2: Already voted for candidate 10b148b3d2c6476e9926cfb65d3fcaad in this term.
23:08:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.418114 25228 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 election: Requested vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.419482 24818 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.420111 24818 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 1 FOLLOWER]: Advancing to term 2
23:08:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.420675 24752 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49
23:08:38.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.422863 25036 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.426995 24818 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 2.
23:08:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.427544 25233 raft_consensus.cc:2749] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:08:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.428360 25037 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:08:38.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.429136 25228 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Leader election won for term 2
23:08:38.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.431141 25228 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 LEADER]: Becoming Leader. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Running, Role: LEADER
23:08:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.432956 25228 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:38.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:38.443508 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad reported cstate change: term changed from 1 to 2, leader changed from 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66) to 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67). New cstate: current_term: 2 leader_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:08:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:38.975244 25036 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:08:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:39.018180 24818 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Refusing update from remote peer 10b148b3d2c6476e9926cfb65d3fcaad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:08:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:39.019780 25252 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Connected to new peer: Peer: permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:08:40.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:40.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:40.699 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3675), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [5ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [5ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [14ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [16ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [26ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [27ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [49ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [55ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [57ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [60ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [60ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [108ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [113ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [116ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [119ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [119ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [215ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [216ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [218ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [218ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [409ms] refreshing cache from master, [409ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [414ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [417ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [419ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [419ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [828ms] refreshing cache from master, [829ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [833ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [835ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [837ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [837ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3669ms] refreshing cache from master, [3669ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [3673ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@431288571(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@e8ec96 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518) -> (continuation of Deferred@2108342237 after retry RPC@1584877535) -> (continuation of Deferred@1317361764 after retry RPC@711524316), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b1edc3 -> wakeup thread Time-limited test -> (continuation of Deferred@433594713 after retry RPC@305574449) -> (continuation of Deferred@1958428883 after retry RPC@1032233668) -> (continuation of Deferred@316179980 after retry RPC@1772754805) -> (continuation of Deferred@1099684917 after retry RPC@511156518) -> (continuation of Deferred@2108342237 after retry RPC@1584877535) -> (continuation of Deferred@1317361764 after retry RPC@711524316))) after lookup
23:08:40.700 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:40.702 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] Successfully connected to peer
23:08:40.721 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xc5494c9c, L:/127.0.0.1:50092 - R:/127.23.220.67:38085] using TOKEN/null
23:08:40.778 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.66:42405
23:08:40.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:40.782809 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.117426 25270 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.118000 25270 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.118757 25270 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.152577 25270 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.153517 25270 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.66
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:41.191246 25270 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:41.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:41.192675 25270 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:41.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:41.194368 25270 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:41.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.209452 25277 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.392002 25036 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:08:42.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:41.209607 25276 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:42.585516 25279 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:42.588191 25278 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1380 milliseconds
23:08:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.588281 25270 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.589776 25270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.592947 25270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.594425 25270 hybrid_clock.cc:648] HybridClock initialized: now 1783465722594248 us; error 197 us; skew 500 ppm
23:08:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.595425 25270 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.607350 25270 webserver.cc:534] Webserver started at http://127.23.220.66:34525/ using document root <none> and password file <none>
23:08:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.608700 25270 fs_manager.cc:362] Metadata directory not provided
23:08:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.609022 25270 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.621693 25270 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.006s	sys 0.000s
23:08:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.626487 25290 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.627839 25270 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.628183 25270 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.630271 25270 fs_report.cc:389] FS layout report
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/data
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.684474 25270 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.686036 25270 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.686501 25270 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.689242 25270 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.696266 25297 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:08:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.705641 25270 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:08:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.705960 25270 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:08:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.706284 25270 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:08:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.711666 25270 ts_tablet_manager.cc:616] Registered 1 tablets
23:08:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.711946 25270 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
23:08:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.712487 25297 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap starting.
23:08:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.773330 25297 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Log is configured to *not* fsync() on all Append() calls
23:08:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.820333 25297 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:08:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.821514 25297 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap complete.
23:08:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.823073 25297 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent bootstrapping tablet: real 0.111s	user 0.068s	sys 0.035s
23:08:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.845474 25297 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.848387 25297 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Initialized, Role: FOLLOWER
23:08:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.849926 25297 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.856253 25297 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent starting tablet: real 0.033s	user 0.028s	sys 0.004s
23:08:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.920414 25270 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.66:42405
23:08:42.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.920763 25405 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.66:42405 every 8 connection(s)
23:08:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.923434 25270 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.926952 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 25270
23:08:42.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:42.963 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:08:42.964 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.67:38085
23:08:42.965 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:08:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.966540 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 25016
23:08:42.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.974020 25407 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.974640 25407 heartbeater.cc:461] Registering TS with master...
23:08:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.976105 25407 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.977932 25406 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.978333 25406 heartbeater.cc:461] Registering TS with master...
23:08:42.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.978404 25408 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:42.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.978785 25408 heartbeater.cc:461] Registering TS with master...
23:08:42.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.979436 25406 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:42.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.979663 25408 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:42.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.981832 24631 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.983547 24555 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.983522 24478 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:42.985710 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.66:34323
23:08:43.005 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] handling channelInactive
23:08:43.005 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state READY due to: connection closed
23:08:43.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:43.007 [DEBUG - Time-limited test] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.008 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.008 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.008 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.018 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:43.018 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:43.019 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test)) after lookup
23:08:43.019 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.020 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.020 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.037 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:43.037 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:43.038 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [13ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [14ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [14ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [31ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572))) after lookup
23:08:43.038 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.040 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.077 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:43.078 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:43.078 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [13ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [14ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [14ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [31ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [32ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [34ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [34ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814))) after lookup
23:08:43.079 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.081 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.218 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:43.219 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:43.219 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=213), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [13ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [14ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [14ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [31ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [32ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [34ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [34ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [73ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [75ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [76ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [212ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339))) after lookup
23:08:43.220 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.222 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.222 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.419 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:43.419 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:43.420 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=413), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [13ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [14ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [14ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [31ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [32ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [34ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [34ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [73ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [75ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [76ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [212ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [214ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [216ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [216ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [406ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [412ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339) -> (continuation of Deferred@757064094 after retry RPC@64107732), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339) -> (continuation of Deferred@757064094 after retry RPC@64107732))) after lookup
23:08:43.421 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:43.422 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.422 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.423 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:43.990003 25407 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:44.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.184391 25418 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:08:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.185002 25418 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:44.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.187583 25418 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:44.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.192049 25292 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.197257 25292 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.200130 24818 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:08:44.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.201915 25293 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49; no voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a
23:08:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.202771 25418 raft_consensus.cc:2749] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:08:44.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.371047 25423 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:08:44.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.371457 25423 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:44.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.373294 25423 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.376340 24752 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.380160 24752 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.385996 25361 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49" is_pre_election: true
23:08:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.386816 25361 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 1.
23:08:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.388223 24752 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:08:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.388942 25423 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Leader pre-election won for term 3
23:08:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.389212 25423 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Starting leader election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:08:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.389446 25423 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 2 FOLLOWER]: Advancing to term 3
23:08:44.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.394323 25423 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.395879 25423 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.397583 25361 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.398226 25361 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 1 FOLLOWER]: Advancing to term 3
23:08:44.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.399318 24752 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.404984 25361 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 3.
23:08:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.405894 24752 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:08:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.406541 25423 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Leader election won for term 3
23:08:44.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.408205 25423 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 LEADER]: Becoming Leader. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Running, Role: LEADER
23:08:44.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.409139 25423 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.421803 24630 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a reported cstate change: term changed from 2 to 3, leader changed from 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67) to 641de1a253394bf595b9bc1c4ede893a (127.23.220.65). New cstate: current_term: 3 leader_uuid: "641de1a253394bf595b9bc1c4ede893a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: HEALTHY } } }
23:08:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:44.852901 24752 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a -> Peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Couldn't send request to peer 10b148b3d2c6476e9926cfb65d3fcaad. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:08:44.958 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:44.958 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:44.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1952), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [2ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [2ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [12ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [13ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [14ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [14ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [27ms] refreshing cache from master, [27ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [31ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [32ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [34ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [34ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [71ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [73ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [75ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [76ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [207ms] refreshing cache from master, [207ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [212ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [214ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [216ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [216ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [406ms] refreshing cache from master, [407ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [412ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [414ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [417ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [417ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1947ms] refreshing cache from master, [1947ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1951ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@542464771(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4daed6ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339) -> (continuation of Deferred@757064094 after retry RPC@64107732) -> (continuation of Deferred@617616492 after retry RPC@909570230), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7afdb41c -> wakeup thread Time-limited test -> (continuation of Deferred@884835579 after retry RPC@603425572) -> (continuation of Deferred@1341251775 after retry RPC@1354820814) -> (continuation of Deferred@695283582 after retry RPC@1060122339) -> (continuation of Deferred@757064094 after retry RPC@64107732) -> (continuation of Deferred@617616492 after retry RPC@909570230))) after lookup
23:08:44.960 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:44.961 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] Successfully connected to peer
23:08:44.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.966065 25361 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Refusing update from remote peer 641de1a253394bf595b9bc1c4ede893a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
23:08:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:44.967628 25423 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Connected to new peer: Peer: permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:08:44.987 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xd0b8e524, L:/127.0.0.1:46268 - R:/127.23.220.65:35727] using TOKEN/null
23:08:45.073 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.67:38085
23:08:45.074 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:45.075783 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:08:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:08:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:45.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.405207 25452 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:45.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.405741 25452 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:45.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.406227 25452 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.440023 25452 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:45.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.440898 25452 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.67
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:45.480881 25452 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:45.482389 25452 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:45.484386 25452 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.497997 25460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.499596 25461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:46.760336 25462 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1260 milliseconds
23:08:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.760396 25452 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:45.509219 25463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:46.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.766323 25452 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:46.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.769160 25452 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.770797 25452 hybrid_clock.cc:648] HybridClock initialized: now 1783465726770724 us; error 84 us; skew 500 ppm
23:08:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.771873 25452 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:46.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.779357 25452 webserver.cc:534] Webserver started at http://127.23.220.67:46209/ using document root <none> and password file <none>
23:08:46.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.780226 25452 fs_manager.cc:362] Metadata directory not provided
23:08:46.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.780424 25452 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.788158 25452 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
23:08:46.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.792946 25473 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.794064 25452 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
23:08:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.794469 25452 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.796478 25452 fs_report.cc:389] FS layout report
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/data
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:46.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.853938 25452 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.855739 25452 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.856292 25452 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:46.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.859133 25452 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:46.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.865794 25481 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:08:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.874855 25452 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:08:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.875175 25452 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.001s
23:08:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.875411 25452 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:08:46.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.880180 25452 ts_tablet_manager.cc:616] Registered 1 tablets
23:08:46.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.880370 25452 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
23:08:46.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.880874 25481 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap starting.
23:08:46.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.941332 25481 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Log is configured to *not* fsync() on all Append() calls
23:08:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.982367 25481 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:08:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.983369 25481 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap complete.
23:08:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:46.984844 25481 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent bootstrapping tablet: real 0.104s	user 0.058s	sys 0.042s
23:08:47.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.007371 25481 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:47.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.011348 25481 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Initialized, Role: FOLLOWER
23:08:47.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.012753 25481 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.018769 25481 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent starting tablet: real 0.034s	user 0.029s	sys 0.006s
23:08:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.077414 25452 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.67:38085
23:08:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.077694 25590 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.67:38085 every 8 connection(s)
23:08:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.080242 25452 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:08:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.081346 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 25452
23:08:47.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:47.148 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:08:47.149 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.65:35727
23:08:47.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:08:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.150214 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 24713
23:08:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.159677 25591 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.160262 25591 heartbeater.cc:461] Registering TS with master...
23:08:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.161763 25591 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.165998 25593 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.166430 25593 heartbeater.cc:461] Registering TS with master...
23:08:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.167043 24555 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.167349 25593 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:47.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.171036 25592 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:47.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.171416 25592 heartbeater.cc:461] Registering TS with master...
23:08:47.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.171775 24478 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.172444 25592 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.175470 24630 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:08:47.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:47.177670 24630 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.67:37129
23:08:47.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:47.190 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] handling channelInactive
23:08:47.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state READY due to: connection closed
23:08:47.191 [DEBUG - Time-limited test] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.193 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.193 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.193 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.218 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:47.219 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:47.219 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test)) after lookup
23:08:47.220 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.221 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.221 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.221 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:47.238 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:47.238 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [28ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [30ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [30ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233))) after lookup
23:08:47.239 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.241 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.241 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.241 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.277 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:47.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:47.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [28ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [30ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [30ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [48ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [50ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [50ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [86ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528))) after lookup
23:08:47.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.281 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.337 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:47.338 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:47.338 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=147), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [28ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [30ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [30ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [48ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [50ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [50ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [86ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [88ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [90ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [90ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [146ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504))) after lookup
23:08:47.339 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.341 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:47.858 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:47.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=667), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [28ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [30ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [30ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [48ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [50ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [50ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [86ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [88ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [90ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [90ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [146ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [148ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [150ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [150ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [661ms] refreshing cache from master, [662ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [666ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504) -> (continuation of Deferred@2126144574 after retry RPC@2107987186), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504) -> (continuation of Deferred@2126144574 after retry RPC@2107987186))) after lookup
23:08:47.859 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:08:47.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.860 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:47.861 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:08:48.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.181082 25592 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.446565 25604 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:08:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.447050 25604 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:48.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.449637 25604 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:48.454892 25476 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:08:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:48.459553 25476 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:08:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.461128 25361 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49" is_pre_election: true
23:08:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.462778 25475 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad; no voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a
23:08:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.463503 25604 raft_consensus.cc:2749] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
23:08:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.564538 25609 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:08:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.564978 25609 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:48.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.566498 25609 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:48.585533 25293 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:08:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.592329 25544 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:08:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.593313 25544 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 2.
23:08:48.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.595007 25292 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:08:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.595682 25609 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:08:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.596012 25609 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:08:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.596238 25609 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 3 FOLLOWER]: Advancing to term 4
23:08:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.600821 25609 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.602625 25609 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.603408 25544 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:08:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.604089 25544 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 2 FOLLOWER]: Advancing to term 4
23:08:48.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:48.607698 25293 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:08:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.611604 25544 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 4.
23:08:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.612713 25292 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:08:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.613474 25609 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Leader election won for term 4
23:08:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.614751 25609 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 LEADER]: Becoming Leader. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Running, Role: LEADER
23:08:48.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.615686 25609 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:48.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:48.625658 24630 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 reported cstate change: term changed from 3 to 4, leader changed from 641de1a253394bf595b9bc1c4ede893a (127.23.220.65) to 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66). New cstate: current_term: 4 leader_uuid: "59052f49be5346a4ba48421e0b75dc49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:08:48.978 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:48.978 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:48.978 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1787), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [27ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [28ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [30ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [30ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [46ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [48ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [50ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [50ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [86ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [88ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [90ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [90ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [146ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [148ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [150ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [150ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [661ms] refreshing cache from master, [662ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [666ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [668ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [670ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [670ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1782ms] refreshing cache from master, [1782ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1786ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1321063767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@4f0f2f76 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504) -> (continuation of Deferred@2126144574 after retry RPC@2107987186) -> (continuation of Deferred@1400698693 after retry RPC@864704818), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b965e79 -> wakeup thread Time-limited test -> (continuation of Deferred@465305907 after retry RPC@64736233) -> (continuation of Deferred@1457114713 after retry RPC@184599528) -> (continuation of Deferred@18055137 after retry RPC@201454504) -> (continuation of Deferred@2126144574 after retry RPC@2107987186) -> (continuation of Deferred@1400698693 after retry RPC@864704818))) after lookup
23:08:48.979 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:48.980 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] Successfully connected to peer
23:08:49.001 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf74a938a, L:/127.0.0.1:54928 - R:/127.23.220.66:42405] using TOKEN/null
23:08:49.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.013236 25544 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Refusing update from remote peer 59052f49be5346a4ba48421e0b75dc49: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
23:08:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.015581 25614 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.001s
23:08:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.019295 25293 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 -> Peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Couldn't send request to peer 641de1a253394bf595b9bc1c4ede893a. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:08:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.036581 25631 mvcc.cc:204] Tried to move back new op lower bound from 7305075609795735552 to 7305075607190274048. Current Snapshot: MvccSnapshot[applied={T|T < 7305075609594548224}]
23:08:49.105 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.65:35727
23:08:49.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.108088 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.449673 25638 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:49.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.450266 25638 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.450919 25638 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.487461 25638 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.488562 25638 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.65
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.527743 25638 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.529765 25638 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:49.532125 25638 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.547052 25645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.556793 25293 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:50.951068 25644 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 25638
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:51.208431 25644 kernel_stack_watchdog.cc:198] Thread 25638 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 398ms:
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:51.209121 25638 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.659s	user 0.488s	sys 1.062s
23:08:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:51.209551 25638 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.660s	user 0.488s	sys 1.062s
23:08:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:49.549964 25646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:51.211695 25648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:51.214567 25647 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1660 milliseconds
23:08:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.214648 25638 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.215895 25638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:51.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.218205 25638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.219612 25638 hybrid_clock.cc:648] HybridClock initialized: now 1783465731219596 us; error 66 us; skew 500 ppm
23:08:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.220510 25638 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.227018 25638 webserver.cc:534] Webserver started at http://127.23.220.65:36877/ using document root <none> and password file <none>
23:08:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.228053 25638 fs_manager.cc:362] Metadata directory not provided
23:08:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.228343 25638 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.236871 25638 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
23:08:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.241888 25663 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:51.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.243160 25638 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:08:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.243599 25638 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.245563 25638 fs_report.cc:389] FS layout report
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/data
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.313402 25638 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.315122 25638 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.315692 25638 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.318620 25638 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.325028 25670 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:08:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.334769 25638 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:08:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.335081 25638 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.001s	sys 0.001s
23:08:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.335458 25638 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:08:51.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.340659 25638 ts_tablet_manager.cc:616] Registered 1 tablets
23:08:51.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.340910 25638 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
23:08:51.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.341298 25670 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap starting.
23:08:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.400322 25670 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Log is configured to *not* fsync() on all Append() calls
23:08:51.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.461751 25670 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:08:51.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.462813 25670 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap complete.
23:08:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.464711 25670 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent bootstrapping tablet: real 0.124s	user 0.082s	sys 0.037s
23:08:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.487643 25670 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.490072 25670 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Initialized, Role: FOLLOWER
23:08:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.491222 25670 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.495986 25670 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent starting tablet: real 0.031s	user 0.029s	sys 0.002s
23:08:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.523696 25638 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.65:35727
23:08:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.523964 25778 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.65:35727 every 8 connection(s)
23:08:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.527448 25638 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:08:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.531901 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 25638
23:08:51.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:51.568 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:08:51.576 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.66:42405
23:08:51.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:08:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.578980 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 25270
23:08:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.597540 25779 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.598116 25779 heartbeater.cc:461] Registering TS with master...
23:08:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.599526 25779 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.597563 25780 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.602854 25780 heartbeater.cc:461] Registering TS with master...
23:08:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.603780 25780 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.604453 24555 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.607165 24631 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.610093 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.65:35637
23:08:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.611742 25784 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.612145 25784 heartbeater.cc:461] Registering TS with master...
23:08:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.613202 25784 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:51.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:51.617432 24477 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:51.627 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] handling channelInactive
23:08:51.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state READY due to: connection closed
23:08:51.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:51.628 [DEBUG - Time-limited test] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:51.629 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.629 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.630 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.637 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:51.638 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:51.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test)) after lookup
23:08:51.638 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:51.640 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.640 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.657 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:51.658 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:51.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086))) after lookup
23:08:51.659 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:51.660 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.660 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.661 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:51.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:51.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396))) after lookup
23:08:51.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:51.700 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.700 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.701 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:51.838 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:51.838 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=210), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [70ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [73ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [73ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [209ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603))) after lookup
23:08:51.839 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:51.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:51.840 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.137 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:52.138 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:52.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=510), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [70ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [73ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [73ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [209ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [210ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [212ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [212ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [504ms] refreshing cache from master, [505ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [509ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615))) after lookup
23:08:52.139 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:52.143 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.143 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.143 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.477 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:52.477 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:52.478 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=850), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [70ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [73ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [73ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [209ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [210ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [212ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [212ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [504ms] refreshing cache from master, [505ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [509ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [511ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [515ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [515ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [845ms] refreshing cache from master, [845ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [849ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061))) after lookup
23:08:52.479 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:52.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.480 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.537 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:52.538 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:52.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=910), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [70ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [73ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [73ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [209ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [210ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [212ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [212ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [504ms] refreshing cache from master, [505ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [509ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [511ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [515ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [515ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [845ms] refreshing cache from master, [845ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [849ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [850ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [852ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [852ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [905ms] refreshing cache from master, [905ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [909ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061) -> (continuation of Deferred@1859378284 after retry RPC@483133346), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061) -> (continuation of Deferred@1859378284 after retry RPC@483133346))) after lookup
23:08:52.539 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:08:52.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.541 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:08:52.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.586674 25791 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:08:52.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.587064 25791 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:52.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.588761 25791 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:52.597178 25475 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:52.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.604738 25733 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:08:52.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.605566 25733 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 3.
23:08:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.606974 25476 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:08:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.607577 25791 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Leader pre-election won for term 5
23:08:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.607882 25791 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Starting leader election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:08:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.608105 25791 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 4 FOLLOWER]: Advancing to term 5
23:08:52.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.612818 25791 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.614316 25791 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 election: Requested vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.615517 25733 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a"
23:08:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.615942 25780 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:52.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.616079 25733 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 3 FOLLOWER]: Advancing to term 5
23:08:52.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:52.618412 25475 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:08:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.623814 25733 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 5.
23:08:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.624919 25476 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:08:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.625792 25791 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Leader election won for term 5
23:08:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.626973 25791 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 LEADER]: Becoming Leader. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Running, Role: LEADER
23:08:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.628340 25791 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:52.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:52.636969 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad reported cstate change: term changed from 4 to 5, leader changed from 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66) to 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67). New cstate: current_term: 5 leader_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:08:53.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:53.097430 25475 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:08:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:53.152796 25775 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf060 after lost signal to thread 25640
23:08:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:53.204349 25733 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Refusing update from remote peer 10b148b3d2c6476e9926cfb65d3fcaad: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
23:08:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:53.207024 25791 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Connected to new peer: Peer: permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.001s
23:08:53.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:53.226912 25810 mvcc.cc:204] Tried to move back new op lower bound from 7305075626019090432 to 7305075624418516992. Current Snapshot: MvccSnapshot[applied={T|T < 7305075626019090432}]
23:08:53.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:53.610348 25475 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:08:54.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:54.099 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:54.099 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=16, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2471), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [4ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [9ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [10ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [12ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [31ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [33ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [33ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [69ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [70ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [73ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [73ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [205ms] refreshing cache from master, [205ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [209ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [210ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [212ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [212ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [504ms] refreshing cache from master, [505ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [509ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [511ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [515ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [515ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [845ms] refreshing cache from master, [845ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [849ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [850ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [852ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [852ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [905ms] refreshing cache from master, [905ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [909ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [911ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [913ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [913ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2465ms] refreshing cache from master, [2465ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [2470ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@2125933799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1f15b9ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061) -> (continuation of Deferred@1859378284 after retry RPC@483133346) -> (continuation of Deferred@1014533447 after retry RPC@1089830013), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d58826e -> wakeup thread Time-limited test -> (continuation of Deferred@1809925078 after retry RPC@1377036086) -> (continuation of Deferred@1247415150 after retry RPC@129859396) -> (continuation of Deferred@667760529 after retry RPC@1273375603) -> (continuation of Deferred@2016798113 after retry RPC@49472615) -> (continuation of Deferred@531549867 after retry RPC@183161061) -> (continuation of Deferred@1859378284 after retry RPC@483133346) -> (continuation of Deferred@1014533447 after retry RPC@1089830013))) after lookup
23:08:54.100 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:54.105 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] Successfully connected to peer
23:08:54.137 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9dc63474, L:/127.0.0.1:32898 - R:/127.23.220.67:38085] using TOKEN/null
23:08:54.205 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.66:42405
23:08:54.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:54.207775 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:08:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.582547 25829 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:08:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.583230 25829 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:08:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.584085 25829 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:08:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.633309 25829 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:08:54.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.634423 25829 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.66
23:08:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:54.673323 25829 tablet_server_runner.cc:78] Tablet server non-default flags:
23:08:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:08:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:08:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:54.675129 25829 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:08:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:54.676966 25829 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:08:54.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.691133 25837 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:55.677435 25475 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:08:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.695631 25840 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:54.693636 25838 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:08:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:56.102093 25839 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1405 milliseconds
23:08:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.102300 25829 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:08:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.109001 25829 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:08:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.112041 25829 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:08:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.113402 25829 hybrid_clock.cc:648] HybridClock initialized: now 1783465736113376 us; error 43 us; skew 500 ppm
23:08:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.114190 25829 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:08:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.120486 25829 webserver.cc:534] Webserver started at http://127.23.220.66:34525/ using document root <none> and password file <none>
23:08:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.121364 25829 fs_manager.cc:362] Metadata directory not provided
23:08:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.121552 25829 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:08:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.129276 25829 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
23:08:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.134306 25853 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:08:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.135660 25829 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:08:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.135998 25829 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.137846 25829 fs_report.cc:389] FS layout report
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/data
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:08:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:08:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:08:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:08:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:08:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.191882 25829 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:08:56.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.193747 25829 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:08:56.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.194298 25829 kserver.cc:163] Server-wide thread pool size limit: 3276
23:08:56.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.197079 25829 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:08:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.203019 25860 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:08:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.212424 25829 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:08:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.212693 25829 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:08:56.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.213033 25829 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:08:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.217917 25829 ts_tablet_manager.cc:616] Registered 1 tablets
23:08:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.218150 25829 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
23:08:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.218605 25860 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap starting.
23:08:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.287909 25860 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Log is configured to *not* fsync() on all Append() calls
23:08:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.377920 25860 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:08:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.379317 25860 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap complete.
23:08:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.381362 25860 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent bootstrapping tablet: real 0.163s	user 0.106s	sys 0.049s
23:08:56.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.401010 25829 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.66:42405
23:08:56.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.401237 25967 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.66:42405 every 8 connection(s)
23:08:56.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.403960 25829 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:08:56.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.404807 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 25829
23:08:56.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:56.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.408555 25860 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:56.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.412358 25860 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Initialized, Role: FOLLOWER
23:08:56.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.413424 25860 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:56.463 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:08:56.464 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.67:38085
23:08:56.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:08:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.465451 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 25452
23:08:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.469388 25969 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:08:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.469681 25968 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:08:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.469934 25969 heartbeater.cc:461] Registering TS with master...
23:08:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.470191 25968 heartbeater.cc:461] Registering TS with master...
23:08:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.471969 25968 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:08:56.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.473698 25860 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent starting tablet: real 0.092s	user 0.045s	sys 0.013s
23:08:56.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.474193 25969 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:08:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.475932 25970 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:08:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.476400 25970 heartbeater.cc:461] Registering TS with master...
23:08:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.477444 25970 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:08:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.478019 24555 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.478472 24631 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:56.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.481338 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.66:57867
23:08:56.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.481401 24477 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:56.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:56.486107 25969 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:08:56.502 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] handling channelInactive
23:08:56.503 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state READY due to: connection closed
23:08:56.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:08:56.504 [DEBUG - Time-limited test] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:56.505 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.505 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.505 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.517 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:56.517 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:56.517 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test)) after lookup
23:08:56.522 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:56.523 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.523 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.524 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.537 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:56.537 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:56.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719))) after lookup
23:08:56.538 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:56.539 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.539 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.540 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.558 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:56.558 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:56.559 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [34ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [36ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [36ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583))) after lookup
23:08:56.559 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:56.561 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.561 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.561 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.657 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:56.657 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:56.658 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [34ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [36ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [36ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [55ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [57ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [57ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745))) after lookup
23:08:56.658 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:56.659 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.659 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.659 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:56.997 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:56.998 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:56.998 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=494), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [34ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [36ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [36ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [55ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [57ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [57ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [154ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [155ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [155ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [489ms] refreshing cache from master, [489ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [493ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795))) after lookup
23:08:56.999 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:57.000 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.000 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.000 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.570461 25980 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:08:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.571223 25980 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:57.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.574342 25980 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:08:57.585 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:57.585 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:08:57.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:57.587009 25855 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:57.590 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1082), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [34ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [36ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [36ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [55ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [57ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [57ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [154ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [155ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [155ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [489ms] refreshing cache from master, [489ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [493ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [494ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [496ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [496ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1069ms] refreshing cache from master, [1071ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1080ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795) -> (continuation of Deferred@1914978119 after retry RPC@773379244), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795) -> (continuation of Deferred@1914978119 after retry RPC@773379244))) after lookup
23:08:57.591 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:08:57.593 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.593 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:08:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.596524 25733 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:08:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:57.598568 25855 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.599033 25855 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49; no voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a
23:08:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.599821 25980 raft_consensus.cc:2749] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
23:08:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.919591 25985 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:08:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.920040 25985 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:57.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.922448 25985 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:57.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:57.927495 25665 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:57.932724 25665 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.936826 25923 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49" is_pre_election: true
23:08:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.937482 25923 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 4.
23:08:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.938879 25665 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:08:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.939512 25985 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Leader pre-election won for term 6
23:08:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.939833 25985 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Starting leader election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:08:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.940078 25985 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 5 FOLLOWER]: Advancing to term 6
23:08:57.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.944643 25985 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:57.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.946277 25985 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:08:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.947834 25923 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49"
23:08:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.948464 25923 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 4 FOLLOWER]: Advancing to term 6
23:08:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:57.950363 25665 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:08:57.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.955718 25923 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 6.
23:08:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.956789 25665 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:08:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.957494 25985 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Leader election won for term 6
23:08:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.959343 25985 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 LEADER]: Becoming Leader. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Running, Role: LEADER
23:08:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.960340 25985 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:08:57.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:57.969367 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a reported cstate change: term changed from 5 to 6, leader changed from 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67) to 641de1a253394bf595b9bc1c4ede893a (127.23.220.65). New cstate: current_term: 6 leader_uuid: "641de1a253394bf595b9bc1c4ede893a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: HEALTHY } } }
23:08:58.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:58.405725 25923 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Refusing update from remote peer 641de1a253394bf595b9bc1c4ede893a: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:08:58.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:08:58.407461 25985 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Connected to new peer: Peer: permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
23:08:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:08:58.419625 25665 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a -> Peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Couldn't send request to peer 10b148b3d2c6476e9926cfb65d3fcaad. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:08:59.996 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:08:59.997 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:08:59.999 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3495), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [13ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [14ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [20ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [20ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [33ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [34ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [36ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [36ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [53ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [55ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [57ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [57ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [153ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [154ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [155ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [155ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [489ms] refreshing cache from master, [489ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [493ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [494ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [496ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [496ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1069ms] refreshing cache from master, [1071ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1080ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [1086ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1089ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1089ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3488ms] refreshing cache from master, [3489ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [3492ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1572652175(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@58242f1b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795) -> (continuation of Deferred@1914978119 after retry RPC@773379244) -> (continuation of Deferred@591874428 after retry RPC@1680287751), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a489ed2 -> wakeup thread Time-limited test -> (continuation of Deferred@1843737917 after retry RPC@1019376719) -> (continuation of Deferred@1668745058 after retry RPC@1287157583) -> (continuation of Deferred@1740647193 after retry RPC@1749891745) -> (continuation of Deferred@1764813462 after retry RPC@1755925795) -> (continuation of Deferred@1914978119 after retry RPC@773379244) -> (continuation of Deferred@591874428 after retry RPC@1680287751))) after lookup
23:08:59.999 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:00.001 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] Successfully connected to peer
23:09:00.019 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8e2610e0, L:/127.0.0.1:38208 - R:/127.23.220.65:35727] using TOKEN/null
23:09:00.076 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.67:38085
23:09:00.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:00.078765 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.395043 26021 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.395619 26021 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:00.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.396098 26021 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.429517 26021 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.430358 26021 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.67
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:00.473222 26021 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:00.474721 26021 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:00.476702 26021 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:00.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.489559 26029 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:00.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.854375 25665 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a -> Peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Couldn't send request to peer 10b148b3d2c6476e9926cfb65d3fcaad. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.494252 26032 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:00.491114 26030 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:01.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:01.828680 26031 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1333 milliseconds
23:09:01.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.828832 26021 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:01.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.830150 26021 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.833381 26021 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.834906 26021 hybrid_clock.cc:648] HybridClock initialized: now 1783465741834833 us; error 94 us; skew 500 ppm
23:09:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.835731 26021 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.842353 26021 webserver.cc:534] Webserver started at http://127.23.220.67:46209/ using document root <none> and password file <none>
23:09:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.843408 26021 fs_manager.cc:362] Metadata directory not provided
23:09:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.843616 26021 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:01.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.852481 26021 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.001s	sys 0.003s
23:09:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.857528 26044 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:01.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.858932 26021 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:09:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.859357 26021 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:09:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:09:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.861379 26021 fs_report.cc:389] FS layout report
23:09:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/data
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:01.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.918392 26021 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:01.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.919991 26021 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:01.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.920419 26021 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:01.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.923043 26021 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:01.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.928865 26051 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.937856 26021 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.938133 26021 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.000s
23:09:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.938372 26021 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.943243 26021 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.943465 26021 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.003s
23:09:01.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:01.944043 26051 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap starting.
23:09:02.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.008414 26051 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Log is configured to *not* fsync() on all Append() calls
23:09:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.083329 26051 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.084304 26051 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap complete.
23:09:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.086124 26051 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent bootstrapping tablet: real 0.143s	user 0.111s	sys 0.024s
23:09:02.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.105604 26051 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:02.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.109120 26051 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Initialized, Role: FOLLOWER
23:09:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.110342 26051 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:02.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.114763 26051 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent starting tablet: real 0.028s	user 0.027s	sys 0.004s
23:09:02.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.138932 26021 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.67:38085
23:09:02.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.139593 26159 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.67:38085 every 8 connection(s)
23:09:02.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.142027 26021 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.143798 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26021
23:09:02.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:02.188 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:02.188 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.65:35727
23:09:02.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:09:02.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.189627 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 25638
23:09:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.207746 26160 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.208293 26160 heartbeater.cc:461] Registering TS with master...
23:09:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.209726 26160 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.212371 26161 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.212723 26161 heartbeater.cc:461] Registering TS with master...
23:09:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.213716 26161 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.215826 24555 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.216513 26162 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.216899 26162 heartbeater.cc:461] Registering TS with master...
23:09:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.217201 24631 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.217875 26162 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.219379 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.67:44287
23:09:02.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:02.222138 24477 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:02.231 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] handling channelInactive
23:09:02.231 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state READY due to: connection closed
23:09:02.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:02.233 [DEBUG - Time-limited test] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.234 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.234 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.234 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:02.258 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:02.258 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test)) after lookup
23:09:02.259 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.260 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.260 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.260 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.277 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:02.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:02.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406))) after lookup
23:09:02.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.280 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:02.297 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:02.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074))) after lookup
23:09:02.297 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.298 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.299 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.299 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:02.398 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:02.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=166), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [65ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [67ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [67ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [165ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053))) after lookup
23:09:02.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.400 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.401 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:02.698 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:02.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=466), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [65ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [67ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [67ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [165ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [167ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [169ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [169ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [465ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971))) after lookup
23:09:02.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:02.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.700 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:02.700 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.158 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:03.158 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:03.158 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=926), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [65ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [67ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [67ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [165ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [167ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [169ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [169ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [465ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [466ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [468ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [468ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [920ms] refreshing cache from master, [921ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [925ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874))) after lookup
23:09:03.159 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:03.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.160 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.160 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.224324 26161 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:03.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.394152 26171 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:09:03.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.394661 26171 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.396188 26171 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:03.414839 25856 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:03.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:03.419008 25856 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.423449 26114 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:09:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.424237 26114 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 5.
23:09:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.425556 25855 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:09:03.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.426285 26171 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:09:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.426656 26171 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:09:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.427120 26171 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 6 FOLLOWER]: Advancing to term 7
23:09:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.433175 26171 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.435626 26171 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.436375 26114 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:09:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.436980 26114 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 5 FOLLOWER]: Advancing to term 7
23:09:03.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:03.438 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:03.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1206), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [65ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [67ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [67ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [165ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [167ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [169ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [169ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [465ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [466ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [468ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [468ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [920ms] refreshing cache from master, [921ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [925ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [927ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [928ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [928ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1201ms] refreshing cache from master, [1201ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1206ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874) -> (continuation of Deferred@1725594019 after retry RPC@728569264), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874) -> (continuation of Deferred@1725594019 after retry RPC@728569264))) after lookup
23:09:03.440 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:03.441 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.441 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.441 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:03.441465 25856 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.445410 26114 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 7.
23:09:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.446431 25855 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:09:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.447206 26171 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Leader election won for term 7
23:09:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.448621 26171 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 LEADER]: Becoming Leader. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Running, Role: LEADER
23:09:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.449852 26171 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.458632 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 reported cstate change: term changed from 6 to 7, leader changed from 641de1a253394bf595b9bc1c4ede893a (127.23.220.65) to 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66). New cstate: current_term: 7 leader_uuid: "59052f49be5346a4ba48421e0b75dc49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:09:03.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.860213 26114 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Refusing update from remote peer 59052f49be5346a4ba48421e0b75dc49: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
23:09:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.861768 26171 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
23:09:03.879 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:03.879 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:03.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=16, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1647), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [2ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [25ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [28ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [28ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [45ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [46ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [48ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [48ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [64ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [65ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [67ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [67ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [165ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [167ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [169ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [169ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [460ms] refreshing cache from master, [461ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [465ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [466ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [468ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [468ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [920ms] refreshing cache from master, [921ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [925ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [927ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [928ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [928ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1201ms] refreshing cache from master, [1201ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1206ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [1207ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1209ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1209ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1641ms] refreshing cache from master, [1641ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1646ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1087407991(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@1d6af87a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874) -> (continuation of Deferred@1725594019 after retry RPC@728569264) -> (continuation of Deferred@1066037224 after retry RPC@2089368892), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dcf8cc8 -> wakeup thread Time-limited test -> (continuation of Deferred@1400550541 after retry RPC@42130406) -> (continuation of Deferred@1051765929 after retry RPC@720969074) -> (continuation of Deferred@921130054 after retry RPC@1410846053) -> (continuation of Deferred@1886371630 after retry RPC@444151971) -> (continuation of Deferred@1495479813 after retry RPC@153655874) -> (continuation of Deferred@1725594019 after retry RPC@728569264) -> (continuation of Deferred@1066037224 after retry RPC@2089368892))) after lookup
23:09:03.881 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:03.887 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] Successfully connected to peer
23:09:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:03.918885 25856 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 -> Peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Couldn't send request to peer 641de1a253394bf595b9bc1c4ede893a. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:09:03.934 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xb241a28d, L:/127.0.0.1:40778 - R:/127.23.220.66:42405] using TOKEN/null
23:09:03.984 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.65:35727
23:09:03.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:03.985796 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:09:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:09:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.327529 26199 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.328130 26199 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.328780 26199 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.365844 26199 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.367038 26199 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.65
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:04.412048 26199 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:04.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:04.414029 26199 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:04.416857 26199 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.433670 26206 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:04.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.967458 25855 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.23.220.67:38085, user_credentials={real_user=slave}} blocked reactor thread for 50320.9us
23:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:04.433988 26207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:05.835469 26205 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 26199
23:09:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:06.239634 26205 kernel_stack_watchdog.cc:198] Thread 26199 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 397ms:
23:09:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:09:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:09:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:09:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:09:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:06.245703 26208 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1804 milliseconds
23:09:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:06.245918 26199 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.811s	user 0.001s	sys 0.001s
23:09:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:06.246304 26199 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.811s	user 0.001s	sys 0.001s
23:09:06.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:06.248467 26209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:06.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.248507 26199 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.249816 26199 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.252070 26199 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.253481 26199 hybrid_clock.cc:648] HybridClock initialized: now 1783465746253431 us; error 47 us; skew 500 ppm
23:09:06.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.254326 26199 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.260527 26199 webserver.cc:534] Webserver started at http://127.23.220.65:36877/ using document root <none> and password file <none>
23:09:06.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.261500 26199 fs_manager.cc:362] Metadata directory not provided
23:09:06.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.261751 26199 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:06.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.270834 26199 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.007s	sys 0.000s
23:09:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.275987 26224 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.277172 26199 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
23:09:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.277585 26199 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:09:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.279729 26199 fs_report.cc:389] FS layout report
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/data
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:06.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.329643 26199 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.331372 26199 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.331934 26199 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.334867 26199 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.340857 26231 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.349681 26199 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.350028 26199 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:09:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.350353 26199 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.355516 26199 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.355767 26199 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
23:09:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.356079 26231 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap starting.
23:09:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.431543 26231 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Log is configured to *not* fsync() on all Append() calls
23:09:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.547389 26199 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.65:35727
23:09:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.547576 26231 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.547523 26338 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.65:35727 every 8 connection(s)
23:09:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.548962 26231 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap complete.
23:09:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.550923 26231 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent bootstrapping tablet: real 0.195s	user 0.129s	sys 0.062s
23:09:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.551093 26199 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:06.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.555966 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26199
23:09:06.557 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:06.595 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:06.597 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.66:42405
23:09:06.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:09:06.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.597936 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 25829
23:09:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.609054 26231 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.640650 26339 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.644816 26340 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.645273 26339 heartbeater.cc:461] Registering TS with master...
23:09:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.645378 26340 heartbeater.cc:461] Registering TS with master...
23:09:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.627632 26231 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Initialized, Role: FOLLOWER
23:09:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.647464 26339 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.646961 26231 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.646982 26340 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.652053 26231 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent starting tablet: real 0.101s	user 0.026s	sys 0.005s
23:09:06.661 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] handling channelInactive
23:09:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.652451 24631 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.655720 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.65:35401
23:09:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.659534 26340 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:06.661 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state READY due to: connection closed
23:09:06.666 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:06.667 [DEBUG - Time-limited test] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:06.668 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.660458 24554 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:06.669 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.669 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.669952 26344 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.670324 26344 heartbeater.cc:461] Registering TS with master...
23:09:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.670946 26344 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:06.674324 24477 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:06.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:06.678 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:06.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test)) after lookup
23:09:06.678 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:06.679 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.679 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.680 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.696 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:06.696 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:06.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [11ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [13ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376))) after lookup
23:09:06.697 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:06.698 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.698 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.698 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:06.717 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:06.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [11ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [13ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [30ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [31ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [31ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [49ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295))) after lookup
23:09:06.718 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:06.719 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.719 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.719 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:06.837 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:06.837 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=170), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [11ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [13ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [30ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [31ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [31ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [49ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [50ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [52ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [52ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [165ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [169ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547))) after lookup
23:09:06.838 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:06.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:06.839 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:07.296 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:07.297 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:07.297 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=630), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [11ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [13ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [30ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [31ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [31ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [49ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [50ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [52ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [52ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [165ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [169ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [170ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [172ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [172ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [626ms] refreshing cache from master, [626ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [629ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547) -> (continuation of Deferred@1361936663 after retry RPC@1546560553), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547) -> (continuation of Deferred@1361936663 after retry RPC@1546560553))) after lookup
23:09:07.298 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:07.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:07.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:07.299 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:07.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.822099 26352 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:09:07.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.822685 26352 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.825331 26352 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:07.831168 26226 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:07.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:07.835914 26226 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.836822 26114 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:09:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.838263 26226 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49
23:09:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.839134 26352 raft_consensus.cc:2749] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
23:09:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.900190 26357 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:09:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.900630 26357 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.903403 26357 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:07.906785 26046 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:07.911752 26046 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.916628 26294 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:09:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.917373 26294 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 6.
23:09:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.918967 26047 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:09:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.919802 26357 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Leader pre-election won for term 8
23:09:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.920140 26357 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Starting leader election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:09:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.920552 26357 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 7 FOLLOWER]: Advancing to term 8
23:09:07.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.925320 26357 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.927105 26357 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 election: Requested vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:07.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.928443 26294 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a"
23:09:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.928959 26294 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 6 FOLLOWER]: Advancing to term 8
23:09:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:07.931120 26046 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.936134 26294 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 8.
23:09:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.937204 26047 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:09:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.937862 26357 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Leader election won for term 8
23:09:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.939705 26357 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 LEADER]: Becoming Leader. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Running, Role: LEADER
23:09:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.940960 26357 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:07.949864 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad reported cstate change: term changed from 7 to 8, leader changed from 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66) to 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67). New cstate: current_term: 8 leader_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:09:08.417 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:08.417 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:08.417 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1750), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [2ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [10ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [11ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [12ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [13ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [29ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [30ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [31ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [31ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [49ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [50ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [52ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [52ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [165ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [169ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [170ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [172ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [172ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [626ms] refreshing cache from master, [626ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [629ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [630ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [632ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [632ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [1745ms] refreshing cache from master, [1745ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1749ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1693620600(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@51273807 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547) -> (continuation of Deferred@1361936663 after retry RPC@1546560553) -> (continuation of Deferred@1130399994 after retry RPC@1783001939), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@45f292a6 -> wakeup thread Time-limited test -> (continuation of Deferred@253247303 after retry RPC@1580137376) -> (continuation of Deferred@728163184 after retry RPC@45940295) -> (continuation of Deferred@473917365 after retry RPC@2031418547) -> (continuation of Deferred@1361936663 after retry RPC@1546560553) -> (continuation of Deferred@1130399994 after retry RPC@1783001939))) after lookup
23:09:08.418 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:08.419 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] Successfully connected to peer
23:09:08.439 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xca5ee30c, L:/127.0.0.1:53038 - R:/127.23.220.67:38085] using TOKEN/null
23:09:08.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:08.450984 26294 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Refusing update from remote peer 10b148b3d2c6476e9926cfb65d3fcaad: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
23:09:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:08.453006 26377 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Connected to new peer: Peer: permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
23:09:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.455772 26046 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:09:08.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:08.470439 26382 mvcc.cc:204] Tried to move back new op lower bound from 7305075687176769536 to 7305075685179695104. Current Snapshot: MvccSnapshot[applied={T|T < 7305075687176769536}]
23:09:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:08.480208 26376 mvcc.cc:204] Tried to move back new op lower bound from 7305075705630896128 to 7305075703574786048. Current Snapshot: MvccSnapshot[applied={T|T < 7305075705630896128}]
23:09:08.565 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.66:42405
23:09:08.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:08.568058 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.931277 26389 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.931802 26389 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.932265 26389 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.969198 26389 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:08.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:08.970204 26389 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.66
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:09.010094 26389 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:09.011567 26389 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:09.013288 26389 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:09.028221 26397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:09.551692 24586 debug-util.cc:398] Leaking SignalData structure 0x7b0800089b40 after lost signal to thread 24519
23:09:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:09.552376 24586 debug-util.cc:398] Leaking SignalData structure 0x7b08000a6be0 after lost signal to thread 24589
23:09:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:10.432498 26396 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 26389
23:09:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:09.031016 26399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:11.013120 26389 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.985s	user 0.619s	sys 1.090s
23:09:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:11.014302 26389 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.986s	user 0.619s	sys 1.090s
23:09:11.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:11.016321 26401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:11.019214 26400 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1985 milliseconds
23:09:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.019246 26389 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.021116 26389 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.023557 26389 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.024978 26389 hybrid_clock.cc:648] HybridClock initialized: now 1783465751024941 us; error 52 us; skew 500 ppm
23:09:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.025938 26389 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.033263 26389 webserver.cc:534] Webserver started at http://127.23.220.66:34525/ using document root <none> and password file <none>
23:09:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.034375 26389 fs_manager.cc:362] Metadata directory not provided
23:09:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.034693 26389 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:11.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.043642 26389 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.007s	sys 0.000s
23:09:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.049050 26412 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:11.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.050647 26389 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.003s
23:09:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.051105 26389 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:09:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.053406 26389 fs_report.cc:389] FS layout report
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/data
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:11.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:11.104470 26046 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:09:11.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.114776 26389 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.117029 26389 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.117633 26389 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.121289 26389 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:11.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.129379 26421 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.143695 26389 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.144035 26389 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.017s	user 0.003s	sys 0.000s
23:09:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.144371 26389 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.152204 26389 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.152518 26389 ts_tablet_manager.cc:595] Time spent register tablets: real 0.008s	user 0.007s	sys 0.000s
23:09:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.153558 26421 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap starting.
23:09:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.241341 26421 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Log is configured to *not* fsync() on all Append() calls
23:09:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.365621 26421 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.366747 26421 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap complete.
23:09:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.370638 26421 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent bootstrapping tablet: real 0.218s	user 0.155s	sys 0.052s
23:09:11.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.392833 26421 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.396703 26421 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Initialized, Role: FOLLOWER
23:09:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.398001 26421 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:11.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.402488 26421 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent starting tablet: real 0.031s	user 0.028s	sys 0.001s
23:09:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.420687 26389 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.66:42405
23:09:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.421103 26529 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.66:42405 every 8 connection(s)
23:09:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.423875 26389 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:11.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.429718 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26389
23:09:11.470 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:11.472 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.67:38085
23:09:11.473 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:09:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.473850 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26021
23:09:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.553715 26530 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.554226 26530 heartbeater.cc:461] Registering TS with master...
23:09:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.555691 26530 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:11.560 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] handling channelInactive
23:09:11.560 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state READY due to: connection closed
23:09:11.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.563572 24554 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:11.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:11.572 [DEBUG - Time-limited test] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.574 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.574 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.574 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.587848 26531 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.588371 26531 heartbeater.cc:461] Registering TS with master...
23:09:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.589598 26531 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:11.597 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.593690 24631 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.596556 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.66:37989
23:09:11.597 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:11.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test)) after lookup
23:09:11.598 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.600 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.619 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:11.620 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:11.620 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372))) after lookup
23:09:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.617255 26535 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.617676 26535 heartbeater.cc:461] Registering TS with master...
23:09:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.618376 26535 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:11.621 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:11.622792 24477 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:11.624 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.624 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.625 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:11.638 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:11.638 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [49ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [53ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [54ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535))) after lookup
23:09:11.639 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.644 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.644 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.644 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:11.678 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:11.679 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=107), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [49ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [53ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [54ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [68ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [73ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [73ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [106ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978))) after lookup
23:09:11.680 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.682 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.682 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.683 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.779 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:11.779 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:11.780 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=208), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [49ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [53ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [54ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [68ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [73ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [73ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [106ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [109ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [111ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [112ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [202ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [207ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091))) after lookup
23:09:11.780 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:11.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:11.782 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.565785 26542 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:09:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.566243 26542 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:12.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.568171 26542 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:12.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:12.590387 26226 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:09:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.594169 26484 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49" is_pre_election: true
23:09:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.595150 26484 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 7.
23:09:12.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:12.598 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:12.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1027), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [49ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [53ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [54ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [68ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [73ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [73ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [106ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [109ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [111ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [112ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [202ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [207ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [209ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [211ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [211ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1022ms] refreshing cache from master, [1022ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1026ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091) -> (continuation of Deferred@1675002164 after retry RPC@987905023), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091) -> (continuation of Deferred@1675002164 after retry RPC@987905023))) after lookup
23:09:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.598543 26226 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:09:12.599 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.599506 26542 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Leader pre-election won for term 9
23:09:12.603 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.599874 26542 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Starting leader election (detected failure of leader 10b148b3d2c6476e9926cfb65d3fcaad)
23:09:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.600262 26542 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 8 FOLLOWER]: Advancing to term 9
23:09:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.603018 26531 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:12.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.607894 26542 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:12.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state CONNECTING due to: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:12.611 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085
23:09:12.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.614472 26542 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:12.616037 26226 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111)
23:09:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.616171 26484 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "641de1a253394bf595b9bc1c4ede893a" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "59052f49be5346a4ba48421e0b75dc49"
23:09:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.616748 26484 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 7 FOLLOWER]: Advancing to term 9
23:09:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.626210 26484 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 641de1a253394bf595b9bc1c4ede893a in term 9.
23:09:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.627386 26226 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 59052f49be5346a4ba48421e0b75dc49, 641de1a253394bf595b9bc1c4ede893a; no voters: 10b148b3d2c6476e9926cfb65d3fcaad
23:09:12.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.628222 26542 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Leader election won for term 9
23:09:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.630328 26542 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 LEADER]: Becoming Leader. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Running, Role: LEADER
23:09:12.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.631522 26542 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:12.640353 24631 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a reported cstate change: term changed from 8 to 9, leader changed from 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67) to 641de1a253394bf595b9bc1c4ede893a (127.23.220.65). New cstate: current_term: 9 leader_uuid: "641de1a253394bf595b9bc1c4ede893a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: HEALTHY } } }
23:09:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:13.041512 26484 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Refusing update from remote peer 641de1a253394bf595b9bc1c4ede893a: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
23:09:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:13.042969 26542 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [LEADER]: Connected to new peer: Peer: permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
23:09:13.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:13.057999 26561 mvcc.cc:204] Tried to move back new op lower bound from 7305075705630896128 to 7305075703574786048. Current Snapshot: MvccSnapshot[applied={T|T < 7305075705630896128}]
23:09:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:13.072374 26226 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111) [suppressed 2 similar messages]
23:09:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:13.076640 26226 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a -> Peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Couldn't send request to peer 10b148b3d2c6476e9926cfb65d3fcaad. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:09:13.537 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:13.538 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:13.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1966), Traces: [0ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [3ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [3ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [26ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [27ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [29ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [29ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [48ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [49ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [53ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [54ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [66ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [68ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [73ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [73ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [102ms] refreshing cache from master, [102ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [106ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [109ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [111ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [112ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [202ms] refreshing cache from master, [202ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [207ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [209ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [211ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [211ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1022ms] refreshing cache from master, [1022ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1026ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [1028ms] sending RPC to server 10b148b3d2c6476e9926cfb65d3fcaad, [1040ms] received response from server 10b148b3d2c6476e9926cfb65d3fcaad: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1040ms] delaying RPC due to: Network error: Failed to connect to peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085): Connection refused: /127.23.220.67:38085, [1962ms] refreshing cache from master, [1962ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1966ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1266685475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@128538eb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091) -> (continuation of Deferred@1675002164 after retry RPC@987905023) -> (continuation of Deferred@775608586 after retry RPC@86778609), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@70f565ec -> wakeup thread Time-limited test -> (continuation of Deferred@257511206 after retry RPC@1415086372) -> (continuation of Deferred@190529097 after retry RPC@463130535) -> (continuation of Deferred@147293714 after retry RPC@1164805978) -> (continuation of Deferred@1641873405 after retry RPC@1210246091) -> (continuation of Deferred@1675002164 after retry RPC@987905023) -> (continuation of Deferred@775608586 after retry RPC@86778609))) after lookup
23:09:13.539 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:13.540 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] Successfully connected to peer
23:09:13.578 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x96895772, L:/127.0.0.1:40460 - R:/127.23.220.65:35727] using TOKEN/null
23:09:13.652 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.67:38085
23:09:13.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:13.653970 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:13.973826 26573 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:13.974336 26573 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:13.974906 26573 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:14.007978 26573 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:14.009378 26573 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.67
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:14.049113 26573 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.67:38085
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.67
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46209
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.67
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/logs
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:14.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:14.050659 26573 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:14.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:14.052455 26573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:14.065757 26579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:15.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.307003 26226 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a -> Peer 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085): Couldn't send request to peer 10b148b3d2c6476e9926cfb65d3fcaad. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.67:38085: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:09:15.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.469856 26578 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 26573
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.774580 26578 kernel_stack_watchdog.cc:198] Thread 26573 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 401ms:
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:14.067127 26580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.775154 26573 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.709s	user 0.495s	sys 1.092s
23:09:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.775622 26573 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.710s	user 0.495s	sys 1.092s
23:09:15.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.777617 26582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:15.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:15.780728 26581 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1708 milliseconds
23:09:15.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.780821 26573 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.782011 26573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:15.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.784217 26573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:15.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.785631 26573 hybrid_clock.cc:648] HybridClock initialized: now 1783465755785600 us; error 43 us; skew 500 ppm
23:09:15.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.786659 26573 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:15.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.794318 26573 webserver.cc:534] Webserver started at http://127.23.220.67:46209/ using document root <none> and password file <none>
23:09:15.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.795473 26573 fs_manager.cc:362] Metadata directory not provided
23:09:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.795756 26573 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:15.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.804543 26573 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.001s	sys 0.004s
23:09:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.810410 26591 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.812248 26573 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.001s
23:09:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.812784 26573 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:09:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:34 on dist-test-slave-dr0q"
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.816139 26573 fs_report.cc:389] FS layout report
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/wal
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/data
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:15.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.891305 26573 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.892968 26573 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.893509 26573 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:15.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.896220 26573 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.902724 26600 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.911422 26573 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.911682 26573 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:09:15.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.911903 26573 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.917340 26573 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.917623 26573 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
23:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.918237 26600 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap starting.
23:09:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:15.994066 26600 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Log is configured to *not* fsync() on all Append() calls
23:09:16.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.099591 26573 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.67:38085
23:09:16.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.100188 26707 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.67:38085 every 8 connection(s)
23:09:16.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.103469 26573 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-2/data/info.pb
23:09:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.105337 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26573
23:09:16.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.145690 26600 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:16.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.147189 26600 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Bootstrap complete.
23:09:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.150254 26600 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent bootstrapping tablet: real 0.233s	user 0.183s	sys 0.040s
23:09:16.164 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:16.165 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.65:35727
23:09:16.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:09:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.166488 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26199
23:09:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.195695 26708 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.195989 26709 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.196300 26708 heartbeater.cc:461] Registering TS with master...
23:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.196564 26709 heartbeater.cc:461] Registering TS with master...
23:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.197929 26708 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.198945 26709 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.201646 26713 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.202045 26713 heartbeater.cc:461] Registering TS with master...
23:09:16.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.203120 26713 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:16.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.198989 26600 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.204047 26600 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Initialized, Role: FOLLOWER
23:09:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.204157 24554 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.205358 26600 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.210800 26600 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad: Time spent starting tablet: real 0.060s	user 0.028s	sys 0.004s
23:09:16.213 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] handling channelInactive
23:09:16.213 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state READY due to: connection closed
23:09:16.215 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:16.216 [DEBUG - Time-limited test] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.217 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.216686 24477 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:16.217 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.218 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.220626 24630 ts_manager.cc:194] Re-registered known tserver with Master: 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085)
23:09:16.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.224789 24630 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.67:33475
23:09:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:16.228509 26709 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:16.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:16.237 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:16.237 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test)) after lookup
23:09:16.238 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.239 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.239 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.239 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.256 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:16.257 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:16.257 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214))) after lookup
23:09:16.257 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.258 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.259 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.259 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.277 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:16.278 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:16.278 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [41ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [43ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [43ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820))) after lookup
23:09:16.279 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.280 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.318 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:16.319 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:16.319 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=103), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [41ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [43ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [43ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [62ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [64ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [64ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [97ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [102ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767))) after lookup
23:09:16.319 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.320 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.321 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:16.697 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:16.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=482), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [41ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [43ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [43ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [62ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [64ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [64ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [97ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [102ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [103ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [105ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [105ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [476ms] refreshing cache from master, [477ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [481ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643))) after lookup
23:09:16.698 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:16.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.700 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:16.700 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:17.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:17.398 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)[L]]}]
23:09:17.398 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1182), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [41ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [43ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [43ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [62ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [64ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [64ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [97ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [102ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [103ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [105ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [105ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [476ms] refreshing cache from master, [477ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [481ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [482ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [484ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [484ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1177ms] refreshing cache from master, [1177ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643) -> (continuation of Deferred@705415585 after retry RPC@1039227256), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643) -> (continuation of Deferred@705415585 after retry RPC@1039227256))) after lookup
23:09:17.399 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] connecting to peer
23:09:17.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:17.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] cleaning up while in state CONNECTING due to: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:17.400 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727
23:09:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.462929 26720 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:09:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.463421 26720 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:17.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.465794 26720 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:17.469987 26416 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:17.474908 26416 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.480448 26663 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" is_pre_election: true
23:09:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.481309 26663 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 8.
23:09:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.482873 26414 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:09:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.483705 26720 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Leader pre-election won for term 10
23:09:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.484033 26720 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 641de1a253394bf595b9bc1c4ede893a)
23:09:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.484272 26720 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 9 FOLLOWER]: Advancing to term 10
23:09:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.490826 26720 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.492517 26720 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 election: Requested vote from peers 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67:38085), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.493407 26663 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "59052f49be5346a4ba48421e0b75dc49" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "10b148b3d2c6476e9926cfb65d3fcaad"
23:09:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.493887 26663 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 8 FOLLOWER]: Advancing to term 10
23:09:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:17.496536 26416 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111)
23:09:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.501094 26663 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59052f49be5346a4ba48421e0b75dc49 in term 10.
23:09:17.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.502215 26414 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 59052f49be5346a4ba48421e0b75dc49; no voters: 641de1a253394bf595b9bc1c4ede893a
23:09:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.502971 26720 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 10 FOLLOWER]: Leader election won for term 10
23:09:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.504667 26720 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 10 LEADER]: Becoming Leader. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Running, Role: LEADER
23:09:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.505661 26720 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.514900 24630 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 reported cstate change: term changed from 9 to 10, leader changed from 641de1a253394bf595b9bc1c4ede893a (127.23.220.65) to 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66). New cstate: current_term: 10 leader_uuid: "59052f49be5346a4ba48421e0b75dc49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:09:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.887872 26663 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Refusing update from remote peer 59052f49be5346a4ba48421e0b75dc49: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
23:09:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:17.889506 26720 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
23:09:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:17.901351 26704 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5060 after lost signal to thread 26574
23:09:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:17.928519 26416 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 -> Peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Couldn't send request to peer 641de1a253394bf595b9bc1c4ede893a. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:09:18.916 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:18.917 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:18.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2701), Traces: [0ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [22ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [23ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [23ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [40ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [41ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [43ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [43ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [61ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [62ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [64ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [64ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [97ms] refreshing cache from master, [97ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [102ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [103ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [105ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [105ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [476ms] refreshing cache from master, [477ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [481ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [482ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [484ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [484ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1177ms] refreshing cache from master, [1177ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [1182ms] sending RPC to server 641de1a253394bf595b9bc1c4ede893a, [1184ms] received response from server 641de1a253394bf595b9bc1c4ede893a: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [1184ms] delaying RPC due to: Network error: Failed to connect to peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727): Connection refused: /127.23.220.65:35727, [2696ms] refreshing cache from master, [2697ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [2700ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@1765257614(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@d682df3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643) -> (continuation of Deferred@705415585 after retry RPC@1039227256) -> (continuation of Deferred@562452039 after retry RPC@378270102), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@46275465 -> wakeup thread Time-limited test -> (continuation of Deferred@562100871 after retry RPC@1568151214) -> (continuation of Deferred@1419881568 after retry RPC@1232149820) -> (continuation of Deferred@417320933 after retry RPC@1493608767) -> (continuation of Deferred@161894529 after retry RPC@1703239643) -> (continuation of Deferred@705415585 after retry RPC@1039227256) -> (continuation of Deferred@562452039 after retry RPC@378270102))) after lookup
23:09:18.917 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:18.920 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] Successfully connected to peer
23:09:18.964 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x041fb6c1, L:/127.0.0.1:35956 - R:/127.23.220.66:42405] using TOKEN/null
23:09:19.031 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.65:35727
23:09:19.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:19.035465 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.570755 26755 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.571947 26755 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.576429 26755 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.623934 26755 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.625075 26755 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.65
23:09:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:19.667384 26755 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.65:35727
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.65
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36877
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.65
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/logs
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:19.669297 26755 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:19.671766 26755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.688798 26762 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:20.297293 26416 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 -> Peer 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727): Couldn't send request to peer 641de1a253394bf595b9bc1c4ede893a. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.65:35727: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:09:21.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:19.690671 26763 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:21.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:21.090811 26761 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 26755
23:09:21.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:21.396116 26755 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.706s	user 0.528s	sys 1.008s
23:09:21.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:21.397478 26755 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.708s	user 0.528s	sys 1.008s
23:09:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:21.401563 26764 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1706 milliseconds
23:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:21.403164 26765 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.403287 26755 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.404981 26755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.407823 26755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:21.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.409411 26755 hybrid_clock.cc:648] HybridClock initialized: now 1783465761409267 us; error 155 us; skew 500 ppm
23:09:21.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.410597 26755 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.419147 26755 webserver.cc:534] Webserver started at http://127.23.220.65:36877/ using document root <none> and password file <none>
23:09:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.420394 26755 fs_manager.cc:362] Metadata directory not provided
23:09:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.420665 26755 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.431788 26755 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.008s	sys 0.000s
23:09:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.436734 26776 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.437951 26755 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.003s
23:09:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.438371 26755 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "641de1a253394bf595b9bc1c4ede893a"
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:28 on dist-test-slave-dr0q"
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.440711 26755 fs_report.cc:389] FS layout report
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/wal
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/data
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.501921 26755 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:21.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.503772 26755 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:21.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.504355 26755 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:21.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.507228 26755 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.513454 26783 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.522038 26755 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.522365 26755 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
23:09:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.522747 26755 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.528198 26755 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.528460 26755 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.004s	sys 0.000s
23:09:21.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.528935 26783 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap starting.
23:09:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.630887 26783 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Log is configured to *not* fsync() on all Append() calls
23:09:21.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.757881 26755 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.65:35727
23:09:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.758667 26890 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.65:35727 every 8 connection(s)
23:09:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.761266 26755 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-0/data/info.pb
23:09:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.771216 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26755
23:09:21.772 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:21.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.804457 26783 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.805648 26783 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Bootstrap complete.
23:09:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.807631 26783 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent bootstrapping tablet: real 0.279s	user 0.188s	sys 0.071s
23:09:21.848 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:21.848 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.23.220.66:42405
23:09:21.849 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:09:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.849684 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26389
23:09:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.846557 26783 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.850600 26893 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.851163 26893 heartbeater.cc:461] Registering TS with master...
23:09:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.852510 26891 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.852751 26893 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.852913 26891 heartbeater.cc:461] Registering TS with master...
23:09:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.853932 26891 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.851166 26783 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 641de1a253394bf595b9bc1c4ede893a, State: Initialized, Role: FOLLOWER
23:09:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.855187 26783 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.859442 26892 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.859947 26892 heartbeater.cc:461] Registering TS with master...
23:09:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.861465 26892 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.867324 24630 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.870327 24477 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.870241 24630 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.65:54105
23:09:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.879045 24554 ts_manager.cc:194] Re-registered known tserver with Master: 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.879563 26783 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a: Time spent starting tablet: real 0.072s	user 0.027s	sys 0.011s
23:09:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:21.879769 26892 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:21.894 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] handling channelInactive
23:09:21.894 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state READY due to: connection closed
23:09:21.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:21.896 [DEBUG - Time-limited test] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:21.897 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.897 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.898 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:21.917 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:21.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test)) after lookup
23:09:21.918 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:21.919 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.919 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.919 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.937 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:21.937 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:21.938 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585))) after lookup
23:09:21.938 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:21.939 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.940 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.977 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:21.977 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:21.977 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [43ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [45ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [45ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421))) after lookup
23:09:21.978 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:21.979 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.979 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:21.979 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.077 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:22.077 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:22.077 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=181), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [43ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [45ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [45ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [83ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [84ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [84ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552))) after lookup
23:09:22.078 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:22.079 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.079 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.079 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:22.197 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:22.198 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=302), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [43ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [45ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [45ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [83ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [84ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [84ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [182ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [184ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [184ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [298ms] refreshing cache from master, [298ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [302ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608))) after lookup
23:09:22.198 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:22.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:22.199 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:23.137 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:23.137 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085),59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)[L],641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:23.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1242), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [43ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [45ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [45ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [83ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [84ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [84ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [182ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [184ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [184ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [298ms] refreshing cache from master, [298ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [302ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [303ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [304ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [304ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [1238ms] refreshing cache from master, [1238ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1242ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608) -> (continuation of Deferred@655275444 after retry RPC@399288440), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608) -> (continuation of Deferred@655275444 after retry RPC@399288440))) after lookup
23:09:23.138 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] connecting to peer
23:09:23.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:23.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] cleaning up while in state CONNECTING due to: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:23.139 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405) for tablet 50caf47eb7d047cf914475445d7f87e2: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405
23:09:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.293269 26904 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:09:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.293674 26904 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.296847 26904 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:23.300266 26595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:23.304677 26595 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.311010 26846 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a" is_pre_election: true
23:09:23.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.311968 26846 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 9.
23:09:23.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.313813 26596 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:09:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.314682 26904 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Leader pre-election won for term 11
23:09:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.315026 26904 raft_consensus.cc:493] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Starting leader election (detected failure of leader 59052f49be5346a4ba48421e0b75dc49)
23:09:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.315259 26904 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 10 FOLLOWER]: Advancing to term 11
23:09:23.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.319962 26904 raft_consensus.cc:515] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:23.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.321568 26904 leader_election.cc:290] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 election: Requested vote from peers 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405), 641de1a253394bf595b9bc1c4ede893a (127.23.220.65:35727)
23:09:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.322885 26846 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "50caf47eb7d047cf914475445d7f87e2" candidate_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "641de1a253394bf595b9bc1c4ede893a"
23:09:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.323500 26846 raft_consensus.cc:3060] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 9 FOLLOWER]: Advancing to term 11
23:09:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:23.325320 26595 leader_election.cc:336] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111)
23:09:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.330582 26846 raft_consensus.cc:2468] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 10b148b3d2c6476e9926cfb65d3fcaad in term 11.
23:09:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.331746 26596 leader_election.cc:304] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 10b148b3d2c6476e9926cfb65d3fcaad, 641de1a253394bf595b9bc1c4ede893a; no voters: 59052f49be5346a4ba48421e0b75dc49
23:09:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.332490 26904 raft_consensus.cc:2804] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 11 FOLLOWER]: Leader election won for term 11
23:09:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.334559 26904 raft_consensus.cc:697] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [term 11 LEADER]: Becoming Leader. State: Replica: 10b148b3d2c6476e9926cfb65d3fcaad, State: Running, Role: LEADER
23:09:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.336251 26904 consensus_queue.cc:237] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.345597 24630 catalog_manager.cc:5696] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad reported cstate change: term changed from 10 to 11, leader changed from 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66) to 10b148b3d2c6476e9926cfb65d3fcaad (127.23.220.67). New cstate: current_term: 11 leader_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } health_report { overall_health: UNKNOWN } } }
23:09:23.598 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50caf47eb7d047cf914475445d7f87e2 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853' with partition [<start>, <end>)
23:09:23.599 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=50caf47eb7d047cf914475445d7f87e2@[10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)[L],59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405),641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)]}]
23:09:23.599 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=50caf47eb7d047cf914475445d7f87e2, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1703), Traces: [0ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [2ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [3ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [21ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [23ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [24ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [24ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [37ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [42ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [43ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [45ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [45ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [81ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [83ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [84ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [84ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [178ms] refreshing cache from master, [178ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [181ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [182ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [184ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [184ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [298ms] refreshing cache from master, [298ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [302ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [303ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [304ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [304ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [1238ms] refreshing cache from master, [1238ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1242ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, [1243ms] sending RPC to server 59052f49be5346a4ba48421e0b75dc49, [1244ms] received response from server 59052f49be5346a4ba48421e0b75dc49: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [1244ms] delaying RPC due to: Network error: Failed to connect to peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405): Connection refused: /127.23.220.66:42405, [1698ms] refreshing cache from master, [1698ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.220.124:40087, [1703ms] Sub RPC GetTableLocations: received response from server master-127.23.220.124:40087: OK, deferred=Deferred@168499737(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fcc6c36b330@43455d8f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608) -> (continuation of Deferred@655275444 after retry RPC@399288440) -> (continuation of Deferred@126906979 after retry RPC@1970111873), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@168f279b -> wakeup thread Time-limited test -> (continuation of Deferred@1451844996 after retry RPC@970273585) -> (continuation of Deferred@1054698556 after retry RPC@771853421) -> (continuation of Deferred@1882664799 after retry RPC@1383577552) -> (continuation of Deferred@1694596527 after retry RPC@178757608) -> (continuation of Deferred@655275444 after retry RPC@399288440) -> (continuation of Deferred@126906979 after retry RPC@1970111873))) after lookup
23:09:23.600 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] connecting to peer
23:09:23.601 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] Successfully connected to peer
23:09:23.626 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x423056a1, L:/127.0.0.1:50888 - R:/127.23.220.67:38085] using TOKEN/null
23:09:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.637243 26846 raft_consensus.cc:1275] T 50caf47eb7d047cf914475445d7f87e2 P 641de1a253394bf595b9bc1c4ede893a [term 11 FOLLOWER]: Refusing update from remote peer 10b148b3d2c6476e9926cfb65d3fcaad: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
23:09:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.639272 26909 consensus_queue.cc:1048] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad [LEADER]: Connected to new peer: Peer: permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.001s
23:09:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:23.640075 26595 consensus_peers.cc:597] T 50caf47eb7d047cf914475445d7f87e2 P 10b148b3d2c6476e9926cfb65d3fcaad -> Peer 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405): Couldn't send request to peer 59052f49be5346a4ba48421e0b75dc49. Status: Network error: Client connection negotiation failed: client connection to 127.23.220.66:42405: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:09:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.666324 26923 mvcc.cc:204] Tried to move back new op lower bound from 7305075767840882688 to 7305075766633250816. Current Snapshot: MvccSnapshot[applied={T|T < 7305075767840882688}]
23:09:23.708 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.23.220.66:42405
23:09:23.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:23.709997 24433 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:09:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:09:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.052373 26932 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:09:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.052873 26932 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:09:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.053337 26932 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:09:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.087399 26932 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:09:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.088245 26932 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.220.66
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:24.131738 26932 tablet_server_runner.cc:78] Tablet server non-default flags:
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.220.84:40159
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.220.66:42405
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.220.66
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34525
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.220.126:38141,127.23.220.125:38335,127.23.220.124:40087
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:09:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.220.66
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/logs
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5090956cd8459f75ecce585332f90a8f50524d7f
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 22:43:31 UTC on e7f111948823
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16829
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:09:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:24.133590 26932 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:09:24.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:24.135407 26932 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:09:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.149114 26939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:24.150164 26940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:25.501828 26942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:09:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:09:25.504599 26941 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1351 milliseconds
23:09:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.504730 26932 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:09:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.505990 26932 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:09:25.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.508421 26932 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:09:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.509842 26932 hybrid_clock.cc:648] HybridClock initialized: now 1783465765509811 us; error 66 us; skew 500 ppm
23:09:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.510821 26932 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:09:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.517401 26932 webserver.cc:534] Webserver started at http://127.23.220.66:34525/ using document root <none> and password file <none>
23:09:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.518386 26932 fs_manager.cc:362] Metadata directory not provided
23:09:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.518698 26932 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:09:25.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.527232 26932 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.005s
23:09:25.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.533701 26953 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:09:25.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.535339 26932 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.004s
23:09:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.535833 26932 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data,/tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59052f49be5346a4ba48421e0b75dc49"
23:09:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:08:31 on dist-test-slave-dr0q"
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.538201 26932 fs_report.cc:389] FS layout report
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/wal
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/data
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:09:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:09:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.607200 26932 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:09:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.609304 26932 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:09:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.609956 26932 kserver.cc:163] Server-wide thread pool size limit: 3276
23:09:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.613585 26932 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:09:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.620069 26960 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:09:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.629001 26932 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:09:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.629268 26932 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.000s
23:09:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.629554 26932 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:09:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.635537 26932 ts_tablet_manager.cc:616] Registered 1 tablets
23:09:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.635753 26932 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
23:09:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.636518 26960 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap starting.
23:09:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.740875 26960 log.cc:826] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Log is configured to *not* fsync() on all Append() calls
23:09:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.832805 26932 rpc_server.cc:307] RPC server started. Bound to: 127.23.220.66:42405
23:09:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.833047 27067 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.220.66:42405 every 8 connection(s)
23:09:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.835829 26932 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskcoHO5i/test-tmp/mini-kudu-cluster7345999923810414354/ts-1/data/info.pb
23:09:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.839867 24433 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu as pid 26932
23:09:25.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:09:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.882889 26960 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:09:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.884135 26960 tablet_bootstrap.cc:492] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Bootstrap complete.
23:09:25.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.886147 26960 ts_tablet_manager.cc:1403] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent bootstrapping tablet: real 0.250s	user 0.201s	sys 0.032s
23:09:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.906441 26960 raft_consensus.cc:359] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:25.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.909833 26960 raft_consensus.cc:740] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59052f49be5346a4ba48421e0b75dc49, State: Initialized, Role: FOLLOWER
23:09:25.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.911067 26960 consensus_queue.cc:260] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "10b148b3d2c6476e9926cfb65d3fcaad" member_type: VOTER last_known_addr { host: "127.23.220.67" port: 38085 } } peers { permanent_uuid: "59052f49be5346a4ba48421e0b75dc49" member_type: VOTER last_known_addr { host: "127.23.220.66" port: 42405 } } peers { permanent_uuid: "641de1a253394bf595b9bc1c4ede893a" member_type: VOTER last_known_addr { host: "127.23.220.65" port: 35727 } }
23:09:25.919 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:25.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.929929 27068 heartbeater.cc:344] Connected to a master server at 127.23.220.125:38335
23:09:25.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.930567 27068 heartbeater.cc:461] Registering TS with master...
23:09:25.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.932307 27068 heartbeater.cc:507] Master 127.23.220.125:38335 requested a full tablet report, sending...
23:09:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.938699 24554 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:25.939 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783465714853, tablet=50caf47eb7d047cf914475445d7f87e2, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:09:25.946 [DEBUG - main] (Connection.java:575) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] disconnecting while in state READY
23:09:25.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.945888 26960 ts_tablet_manager.cc:1434] T 50caf47eb7d047cf914475445d7f87e2 P 59052f49be5346a4ba48421e0b75dc49: Time spent starting tablet: real 0.059s	user 0.036s	sys 0.021s
23:09:25.947 [DEBUG - main] (Connection.java:575) [peer master-127.23.220.126:38141(127.23.220.126:38141)] disconnecting while in state READY
23:09:25.947 [DEBUG - main] (Connection.java:575) [peer master-127.23.220.125:38335(127.23.220.125:38335)] disconnecting while in state READY
23:09:25.947 [DEBUG - main] (Connection.java:575) [peer 641de1a253394bf595b9bc1c4ede893a(127.23.220.65:35727)] disconnecting while in state TERMINATED
23:09:25.947 [DEBUG - main] (Connection.java:575) [peer master-127.23.220.124:40087(127.23.220.124:40087)] disconnecting while in state READY
23:09:25.947 [DEBUG - main] (Connection.java:575) [peer 59052f49be5346a4ba48421e0b75dc49(127.23.220.66:42405)] disconnecting while in state TERMINATED
23:09:25.951 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] handling channelInactive
23:09:25.951 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.220.124:40087(127.23.220.124:40087)] handling channelInactive
23:09:25.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 10b148b3d2c6476e9926cfb65d3fcaad(127.23.220.67:38085)] cleaning up while in state READY due to: connection closed
23:09:25.951 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.23.220.125:38335(127.23.220.125:38335)] handling channelInactive
23:09:25.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.220.125:38335(127.23.220.125:38335)] cleaning up while in state READY due to: connection closed
23:09:25.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:09:25.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.220.124:40087(127.23.220.124:40087)] cleaning up while in state READY due to: connection closed
23:09:25.952 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.220.126:38141(127.23.220.126:38141)] handling channelInactive
23:09:25.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.220.126:38141(127.23.220.126:38141)] cleaning up while in state READY due to: connection closed
23:09:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.953330 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26755
23:09:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.960199 27070 heartbeater.cc:344] Connected to a master server at 127.23.220.126:38141
23:09:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.960597 27070 heartbeater.cc:461] Registering TS with master...
23:09:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.961601 27070 heartbeater.cc:507] Master 127.23.220.126:38141 requested a full tablet report, sending...
23:09:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.965379 24478 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.965710 27069 heartbeater.cc:344] Connected to a master server at 127.23.220.124:40087
23:09:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.966048 27069 heartbeater.cc:461] Registering TS with master...
23:09:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.967047 27069 heartbeater.cc:507] Master 127.23.220.124:40087 requested a full tablet report, sending...
23:09:25.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.972859 24631 ts_manager.cc:194] Re-registered known tserver with Master: 59052f49be5346a4ba48421e0b75dc49 (127.23.220.66:42405)
23:09:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.976082 24631 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.220.66:50233
23:09:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.980064 27069 heartbeater.cc:499] Master 127.23.220.124:40087 was elected leader, sending a full tablet report...
23:09:25.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:25.995906 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26932
23:09:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:26.023882 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 26573
23:09:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:26.056090 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 24445
23:09:26.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:26.090421 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 24515
23:09:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:09:26.126622 24433 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskcoHO5i/build/tsan/bin/kudu with pid 24592
23:09:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:09:27Z chronyd exiting

Time: 70.721

OK (1 test)

